Check if file size small enough
This commit is contained in:
parent
9fa95aba03
commit
a09bc1da54
10
main.py
10
main.py
|
@ -9,6 +9,7 @@ log.info("Shammer is getting ready...")
|
||||||
|
|
||||||
# Init static vars
|
# Init static vars
|
||||||
INDEX_URL = "https://bitscuit.be/"
|
INDEX_URL = "https://bitscuit.be/"
|
||||||
|
MAX_CONTENT_LENGTH = 500000
|
||||||
|
|
||||||
# Create session
|
# Create session
|
||||||
session = requests.Session()
|
session = requests.Session()
|
||||||
|
@ -27,7 +28,16 @@ while not urls.empty():
|
||||||
# Perform request
|
# Perform request
|
||||||
log.info("Fetching '%s'..."%url)
|
log.info("Fetching '%s'..."%url)
|
||||||
try:
|
try:
|
||||||
|
r = session.get(url, stream=True)
|
||||||
|
|
||||||
|
# Check file size
|
||||||
|
if ("Content-Length" in r.headers.keys() and int(r.headers["Content-Length"]) > MAX_CONTENT_LENGTH) or ("content-length" in r.headers.keys() and int(r.headers["content-length"]) > MAX_CONTENT_LENGTH):
|
||||||
|
log.info("too large response")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Download full
|
||||||
r = session.get(url)
|
r = session.get(url)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.info("failed")
|
log.info("failed")
|
||||||
log.info(e)
|
log.info(e)
|
||||||
|
|
Loading…
Reference in New Issue