From 4d88f14b9da967ff4f4ef0594cf2dce4ec18aff4 Mon Sep 17 00:00:00 2001 From: bitscuit Date: Fri, 14 May 2021 09:03:12 +0000 Subject: [PATCH] Fixed little issue and moved delay to beginning of loop --- main.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index 3f0fb13..740c080 100644 --- a/main.py +++ b/main.py @@ -82,6 +82,11 @@ while not urls.empty(): time.sleep(60) continue + # Wait random time + if REQUESTS_PER_MINUTE != 0: + l = random.uniform(0, 2*60/REQUESTS_PER_MINUTE) + time.sleep(l) + # Get next url from queue url = urls.get() @@ -109,12 +114,11 @@ while not urls.empty(): # Download full r = session.get(url, timeout=5) + r.close() except Exception as e: log.info("failed") log.info(e) - finally: - r.close() # Filter page for hrefs timeStart = time.perf_counter() @@ -144,9 +148,4 @@ while not urls.empty(): numAdded += 1 log.info("%d urls, %d new, queue length %d"%(len(hrefs), numAdded, urls.qsize())) - # Wait random time - if REQUESTS_PER_MINUTE != 0: - l = random.uniform(0, 2*60/REQUESTS_PER_MINUTE) - time.sleep(l) -