Added counter for number of fetched sites
This commit is contained in:
parent
2d399a5325
commit
9850fdf2b3
4
main.py
4
main.py
|
@ -70,6 +70,7 @@ urls.put(INDEX_URL)
|
||||||
|
|
||||||
# Create list to store visited sites
|
# Create list to store visited sites
|
||||||
visited = set([INDEX_URL])
|
visited = set([INDEX_URL])
|
||||||
|
numFetched = 0
|
||||||
|
|
||||||
# Loop
|
# Loop
|
||||||
while not urls.empty():
|
while not urls.empty():
|
||||||
|
@ -85,7 +86,8 @@ while not urls.empty():
|
||||||
url = urls.get()
|
url = urls.get()
|
||||||
|
|
||||||
# Perform request
|
# Perform request
|
||||||
log.info("Fetching '%s'..."%url)
|
log.info("Fetch %d: '%s'..."%(numFetched, url))
|
||||||
|
numFetched += 1
|
||||||
try:
|
try:
|
||||||
r = session.get(url, stream=True, timeout=5)
|
r = session.get(url, stream=True, timeout=5)
|
||||||
r.close()
|
r.close()
|
||||||
|
|
Loading…
Reference in New Issue