bitscuit
/
Shammer
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added sleep

This commit is contained in:
bitscuit 2021-05-08 21:28:13 +02:00
parent 52b3612a8d
commit 477814c480
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
main.py
View File

@ -2,6 +2,8 @@ import requests
import queue import queue
import re import re
import logging as log import logging as log
import time
import random
# Init logging # Init logging
log.basicConfig(level=log.INFO, format="%(asctime)-15s %(levelname)-8s %(message)s") log.basicConfig(level=log.INFO, format="%(asctime)-15s %(levelname)-8s %(message)s")
@ -10,6 +12,7 @@ log.info("Shammer is getting ready...")
# Init static vars # Init static vars
INDEX_URL = "https://bitscuit.be/" INDEX_URL = "https://bitscuit.be/"
MAX_CONTENT_LENGTH = 500000 MAX_CONTENT_LENGTH = 500000
REQUESTS_PER_MINUTE = 10
# Create session # Create session
session = requests.Session() session = requests.Session()
@ -68,3 +71,7 @@ while not urls.empty():
numAdded += 1 numAdded += 1
log.info("%d urls, %d new, queue length %d"%(len(hrefs), numAdded, urls.qsize())) log.info("%d urls, %d new, queue length %d"%(len(hrefs), numAdded, urls.qsize()))
# Wait random time
l = random.uniform(0, 2*60/REQUESTS_PER_MINUTE)
time.sleep(l)