bitscuit
/
Shammer
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added counter for number of fetched sites

This commit is contained in:
bitscuit 2021-05-09 13:19:27 +02:00
parent 2d399a5325
commit 9850fdf2b3
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
main.py
View File

@ -70,6 +70,7 @@ urls.put(INDEX_URL)
# Create list to store visited sites # Create list to store visited sites
visited = set([INDEX_URL]) visited = set([INDEX_URL])
numFetched = 0
# Loop # Loop
while not urls.empty(): while not urls.empty():
@ -85,7 +86,8 @@ while not urls.empty():
url = urls.get() url = urls.get()
# Perform request # Perform request
log.info("Fetching '%s'..."%url) log.info("Fetch %d: '%s'..."%(numFetched, url))
numFetched += 1
try: try:
r = session.get(url, stream=True, timeout=5) r = session.get(url, stream=True, timeout=5)
r.close() r.close()