Seperate download function
download function has been seperated from get_thread_links and get_image_links
This commit is contained in:
parent
7ab6d2911f
commit
d58d029202
1 changed files with 16 additions and 19 deletions
35
download.py
35
download.py
|
@ -1,5 +1,3 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
######################################################################
|
######################################################################
|
||||||
# Copyright 2009, 2010 ryuslash
|
# Copyright 2009, 2010 ryuslash
|
||||||
#
|
#
|
||||||
|
@ -49,14 +47,8 @@ class Downloader(object):
|
||||||
def set_on_downloaded(self, on_downloaded):
|
def set_on_downloaded(self, on_downloaded):
|
||||||
self.on_downloaded = on_downloaded
|
self.on_downloaded = on_downloaded
|
||||||
|
|
||||||
def get_thread_links(self, baseurl):
|
def download(self, url):
|
||||||
myparser = htmlparser.MyParser()
|
f = None
|
||||||
i = 0
|
|
||||||
code = 0
|
|
||||||
url = None
|
|
||||||
|
|
||||||
while code != 404:
|
|
||||||
url = baseurl + str(i)
|
|
||||||
tries = 10
|
tries = 10
|
||||||
while tries > 0:
|
while tries > 0:
|
||||||
try:
|
try:
|
||||||
|
@ -65,6 +57,18 @@ class Downloader(object):
|
||||||
except IOError:
|
except IOError:
|
||||||
tries -= 1
|
tries -= 1
|
||||||
write("\rTry of %s failed, %d tries left" % (url, tries))
|
write("\rTry of %s failed, %d tries left" % (url, tries))
|
||||||
|
return f
|
||||||
|
|
||||||
|
def get_thread_links(self, baseurl):
|
||||||
|
myparser = htmlparser.MyParser()
|
||||||
|
i = 0
|
||||||
|
code = 0
|
||||||
|
url = None
|
||||||
|
|
||||||
|
while code != 404:
|
||||||
|
url = baseurl + str(i)
|
||||||
|
f = self.download(url)
|
||||||
|
|
||||||
if not f is None:
|
if not f is None:
|
||||||
code = f.getcode()
|
code = f.getcode()
|
||||||
if code == 404:
|
if code == 404:
|
||||||
|
@ -95,15 +99,8 @@ class Downloader(object):
|
||||||
progress.show_progress(i)
|
progress.show_progress(i)
|
||||||
|
|
||||||
img_url = baseurl + link
|
img_url = baseurl + link
|
||||||
tries = 10
|
f = self.download(img_url)
|
||||||
while tries > 0:
|
|
||||||
try:
|
|
||||||
f = urllib.urlopen(img_url)
|
|
||||||
break
|
|
||||||
except IOError:
|
|
||||||
tries -= 1
|
|
||||||
write("\rTry of %s failed, %d tries left" \
|
|
||||||
% (img_url, tries))
|
|
||||||
if not f is None:
|
if not f is None:
|
||||||
s = f.read()
|
s = f.read()
|
||||||
f.close()
|
f.close()
|
||||||
|
|
Loading…
Reference in a new issue