diff --git a/main.py b/main.py index 5906d81..0741567 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,7 @@ import logging as log import time import random import json +import datetime # Init logging log.basicConfig(level=log.INFO, format="%(asctime)-15s %(levelname)-8s %(message)s") @@ -12,7 +13,7 @@ log.info("Shammer is getting ready...") # Init config try: - log.info("Loading 'config.json'...") + log.debug("Loading 'config.json'...") f = open("config.json", "r") config = json.load(f) f.close() @@ -21,7 +22,7 @@ except IOError as e: log.warn("Couldn't open 'config.json'") log.warn("%s"%e) log.info("Creating default configuration file...") - config = {"INDEX_URL":"https://bitscuit.be/", "MAX_CONTENT_LENGTH":500000, "REQUESTS_PER_MINUTE":10} + config = {"INDEX_URL":"https://bitscuit.be/", "MAX_CONTENT_LENGTH":500000, "REQUESTS_PER_MINUTE":10, "HOUR_START":7, "HOUR_STOP":23} f = open("config.json", "w") json.dump(config, f, indent="\t") f.close() @@ -37,6 +38,8 @@ except Exception as e: INDEX_URL = config["INDEX_URL"] MAX_CONTENT_LENGTH = config["MAX_CONTENT_LENGTH"] REQUESTS_PER_MINUTE = config["REQUESTS_PER_MINUTE"] +HOUR_START = config["HOUR_START"] +HOUR_STOP = config["HOUR_STOP"] # Create session session = requests.Session() @@ -50,6 +53,15 @@ visited = set([INDEX_URL]) # Loop while not urls.empty(): + + # Check time + hour = datetime.datetime.now().hour + if not (hour >= HOUR_START and hour < HOUR_STOP): + log.debug("Not right time yet, sleeping") + time.sleep(60) + continue + + # Get next url from queue url = urls.get() # Perform request @@ -96,6 +108,7 @@ while not urls.empty(): log.info("%d urls, %d new, queue length %d"%(len(hrefs), numAdded, urls.qsize())) # Wait random time - l = random.uniform(0, 2*60/REQUESTS_PER_MINUTE) - time.sleep(l) + if REQUESTS_PER_MINUTE != 0: + l = random.uniform(0, 2*60/REQUESTS_PER_MINUTE) + time.sleep(l)