bitscuit
/
Shammer
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Check if file size small enough

This commit is contained in:
bitscuit 2021-05-08 20:58:00 +02:00
parent 9fa95aba03
commit a09bc1da54
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
main.py
View File

@ -9,6 +9,7 @@ log.info("Shammer is getting ready...")
# Init static vars # Init static vars
INDEX_URL = "https://bitscuit.be/" INDEX_URL = "https://bitscuit.be/"
MAX_CONTENT_LENGTH = 500000
# Create session # Create session
session = requests.Session() session = requests.Session()
@ -27,7 +28,16 @@ while not urls.empty():
# Perform request # Perform request
log.info("Fetching '%s'..."%url) log.info("Fetching '%s'..."%url)
try: try:
r = session.get(url, stream=True)
# Check file size
if ("Content-Length" in r.headers.keys() and int(r.headers["Content-Length"]) > MAX_CONTENT_LENGTH) or ("content-length" in r.headers.keys() and int(r.headers["content-length"]) > MAX_CONTENT_LENGTH):
log.info("too large response")
continue
# Download full
r = session.get(url) r = session.get(url)
except Exception as e: except Exception as e:
log.info("failed") log.info("failed")
log.info(e) log.info(e)