ryuslash/4grab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Seperate download function

Browse source 
download function has been seperated from get_thread_links and get_image_links
This commit is contained in:
ryuslash 2010-03-30 15:59:15 +02:00
parent 7ab6d2911f
commit d58d029202
1 changed files with 16 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
download.py
View file

@ -1,5 +1,3 @@
#!/usr/bin/env python
######################################################################
# Copyright 2009, 2010 ryuslash
#
@ -49,6 +47,18 @@ class Downloader(object):
def set_on_downloaded(self, on_downloaded):
self.on_downloaded = on_downloaded
def download(self, url):
f = None
tries = 10
while tries > 0:
try:
f = urllib.urlopen(url)
break
except IOError:
tries -= 1
write("\rTry of %s failed, %d tries left" % (url, tries))
return f
def get_thread_links(self, baseurl):
myparser = htmlparser.MyParser()
i = 0
@ -57,14 +67,8 @@ class Downloader(object):
while code != 404:
url = baseurl + str(i)
tries = 10
while tries > 0:
try:
f = urllib.urlopen(url)
break
except IOError:
tries -= 1
write("\rTry of %s failed, %d tries left" % (url, tries))
f = self.download(url)
if not f is None:
code = f.getcode()
if code == 404:
@ -95,15 +99,8 @@ class Downloader(object):
progress.show_progress(i)
img_url = baseurl + link
tries = 10
while tries > 0:
try:
f = urllib.urlopen(img_url)
break
except IOError:
tries -= 1
write("\rTry of %s failed, %d tries left" \
% (img_url, tries))
f = self.download(img_url)
if not f is None:
s = f.read()
f.close()