Added --thread
With --thread a signle thread ID or thread URL can be entered. If thread is a URL, it will download it. If thread is an ID, a category must also be set.
This commit is contained in:
parent
420bc469a4
commit
f09ea95c84
3 changed files with 43 additions and 5 deletions
15
4grab.py
15
4grab.py
|
@ -25,6 +25,7 @@ import sys
|
||||||
import config
|
import config
|
||||||
import download
|
import download
|
||||||
|
|
||||||
|
base_url = "http://boards.4chan.org/"
|
||||||
parser = optparse.OptionParser()
|
parser = optparse.OptionParser()
|
||||||
|
|
||||||
parser.set_usage(
|
parser.set_usage(
|
||||||
|
@ -36,6 +37,7 @@ This is free software, and you are welcome to redistribute it
|
||||||
under certain conditions.""")
|
under certain conditions.""")
|
||||||
parser.add_option("-e", nargs=2, dest="confval", metavar="CONF VALUE", help="Set configuration option CONF to be VALUE")
|
parser.add_option("-e", nargs=2, dest="confval", metavar="CONF VALUE", help="Set configuration option CONF to be VALUE")
|
||||||
parser.add_option("-c", "--category", dest="tempcat", metavar="CATEGORY", help="Set the category to CATEGORY only for this run")
|
parser.add_option("-c", "--category", dest="tempcat", metavar="CATEGORY", help="Set the category to CATEGORY only for this run")
|
||||||
|
parser.add_option("-t", "--thread", dest="thread", metavar="THREAD", help="""Download only THREAD. If THREAD is only an ID, CATEGORY must also be set. Otherwise, no problem :-)""")
|
||||||
|
|
||||||
(options, args) = parser.parse_args()
|
(options, args) = parser.parse_args()
|
||||||
|
|
||||||
|
@ -51,10 +53,21 @@ if options.confval:
|
||||||
config.Configuration().set_option(options.confval[0], options.confval[1])
|
config.Configuration().set_option(options.confval[0], options.confval[1])
|
||||||
config.Configuration().save()
|
config.Configuration().save()
|
||||||
exit(0)
|
exit(0)
|
||||||
|
elif options.thread:
|
||||||
|
if options.thread[:7] == "http://":
|
||||||
|
t = download.get_image_links("", [options.thread])
|
||||||
|
elif options.tempcat:
|
||||||
|
url = "%s%s/res/" % (base_url, options.tempcat)
|
||||||
|
t = download.get_image_links(url, [options.thread])
|
||||||
|
else:
|
||||||
|
print "if THREAD is not an absolute URL, CATEGORY must also be specified"
|
||||||
|
exit(1)
|
||||||
|
download.get_images(t)
|
||||||
|
exit(0)
|
||||||
elif options.tempcat:
|
elif options.tempcat:
|
||||||
config.Configuration().set_category(options.tempcat)
|
config.Configuration().set_category(options.tempcat)
|
||||||
|
|
||||||
base_url = "http://boards.4chan.org/%s/" % (config.Configuration().get_category())
|
base_url = "%s%s/" % (base_url, config.Configuration().get_category())
|
||||||
|
|
||||||
t = download.get_thread_links(base_url)
|
t = download.get_thread_links(base_url)
|
||||||
t = download.get_image_links(base_url, t)
|
t = download.get_image_links(base_url, t)
|
||||||
|
|
|
@ -58,6 +58,8 @@ def get_thread_links(baseurl):
|
||||||
else:
|
else:
|
||||||
"\rOpening of", url, "did not succeed, trying next one..."
|
"\rOpening of", url, "did not succeed, trying next one..."
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
|
progress.complete()
|
||||||
return myparser.get_hyperlinks()
|
return myparser.get_hyperlinks()
|
||||||
|
|
||||||
def get_image_links(baseurl, t = []):
|
def get_image_links(baseurl, t = []):
|
||||||
|
@ -87,6 +89,7 @@ def get_image_links(baseurl, t = []):
|
||||||
print "\rOpening of", img_url, "did not succeed, trying next one..."
|
print "\rOpening of", img_url, "did not succeed, trying next one..."
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
|
progress.complete()
|
||||||
return mysubparser.get_hyperlinks()
|
return mysubparser.get_hyperlinks()
|
||||||
|
|
||||||
def get_images(t = []):
|
def get_images(t = []):
|
||||||
|
@ -109,6 +112,8 @@ def get_images(t = []):
|
||||||
print "\rNot downloading", link, "already downloaded"
|
print "\rNot downloading", link, "already downloaded"
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
|
progress.complete()
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
# Get a file-like object for the 4chan.org w/imgboard
|
# Get a file-like object for the 4chan.org w/imgboard
|
||||||
base_url = "http://boards.4chan.org/" + config.Configuration().get_category() + "/"
|
base_url = "http://boards.4chan.org/" + config.Configuration().get_category() + "/"
|
||||||
|
|
|
@ -30,15 +30,35 @@ class Progress():
|
||||||
|
|
||||||
self.show_progress(0)
|
self.show_progress(0)
|
||||||
|
|
||||||
|
def __get_true_maxwidth(self, vallen, maxvallen):
|
||||||
|
return self.maxwidth - 4 - vallen - maxvallen
|
||||||
|
|
||||||
def show_progress(self, value):
|
def show_progress(self, value):
|
||||||
str_value = str(value)
|
str_value = str(value)
|
||||||
str_maxvalue = str(self.maxvalue)
|
str_maxvalue = str(self.maxvalue)
|
||||||
true_maxwidth = self.maxwidth - 4 - len(str_value) - len(str_maxvalue)
|
#true_maxwidth = self.maxwidth - 4 - len(str_value) - len(str_maxvalue)
|
||||||
progress = int(round((true_maxwidth/float(self.maxvalue))*value))
|
true_maxwidth = self.__get_true_maxwidth(len(str_value), len(str_maxvalue))
|
||||||
|
#print true_maxwidth, ":", self.maxvalue, ":", value # for debugging purposes
|
||||||
|
|
||||||
|
if self.maxvalue == 0:
|
||||||
|
progress = true_maxwidth
|
||||||
|
else:
|
||||||
|
progress = int(round((true_maxwidth/float(self.maxvalue))*value))
|
||||||
|
|
||||||
|
#self.fd.write("\r%s/%s [%s%s]" % (str_value, str_maxvalue, self.prog_char * progress, self.fill_char * (true_maxwidth - progress)))
|
||||||
|
#self.fd.flush()
|
||||||
|
self.__write_progress(str_value, str_maxvalue, progress, true_maxwidth)
|
||||||
|
|
||||||
|
def __write_progress(self, str_value, str_maxvalue, progress, true_maxwidth):
|
||||||
self.fd.write("\r%s/%s [%s%s]" % (str_value, str_maxvalue, self.prog_char * progress, self.fill_char * (true_maxwidth - progress)))
|
self.fd.write("\r%s/%s [%s%s]" % (str_value, str_maxvalue, self.prog_char * progress, self.fill_char * (true_maxwidth - progress)))
|
||||||
self.fd.flush()
|
self.fd.flush()
|
||||||
if value == self.maxvalue:
|
|
||||||
self.fd.write("\n")
|
def complete(self):
|
||||||
|
str_maxvalue = str(self.maxvalue)
|
||||||
|
vallen = len(str_maxvalue)
|
||||||
|
true_maxwidth = self.__get_true_maxwidth(vallen, vallen)
|
||||||
|
self.__write_progress(str_maxvalue, str_maxvalue, true_maxwidth, true_maxwidth)
|
||||||
|
self.fd.write("\n")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
prog = Progress(200)
|
prog = Progress(200)
|
||||||
|
|
Loading…
Reference in a new issue