Added counter for number of fetched sites
This commit is contained in:
parent
2d399a5325
commit
9850fdf2b3
4
main.py
4
main.py
|
@ -70,6 +70,7 @@ urls.put(INDEX_URL)
|
|||
|
||||
# Create list to store visited sites
|
||||
visited = set([INDEX_URL])
|
||||
numFetched = 0
|
||||
|
||||
# Loop
|
||||
while not urls.empty():
|
||||
|
@ -85,7 +86,8 @@ while not urls.empty():
|
|||
url = urls.get()
|
||||
|
||||
# Perform request
|
||||
log.info("Fetching '%s'..."%url)
|
||||
log.info("Fetch %d: '%s'..."%(numFetched, url))
|
||||
numFetched += 1
|
||||
try:
|
||||
r = session.get(url, stream=True, timeout=5)
|
||||
r.close()
|
||||
|
|
Loading…
Reference in New Issue