diff --git a/4grab.py b/4grab.py index f2b8da8..2cb8a3f 100755 --- a/4grab.py +++ b/4grab.py @@ -23,6 +23,7 @@ import optparse import sys import config + import download import progressbar @@ -31,15 +32,14 @@ def raw_input_with_default(default, prompt): if inp == "": return default return inp +config._optioncreator = raw_input_with_default base_url = "http://boards.4chan.org/" parser = optparse.OptionParser() downloader = download.Downloader(progressbar.Progress) -conf = config.Configuration() - -conf.set_optioncreator(raw_input_with_default) def walk_with_wizard(baseurl): + conf = config.Configuration() wzrd_msg = "Pilates! *SHAZAM* Here they come!" print "Alright, let me put on my robe and wizard hat." @@ -63,8 +63,8 @@ def walk_with_wizard(baseurl): [thread]) else: inp = raw_input("Which category would you like to download? ") - conf.set_category(inp) - baseurl = "%s%s/" % (baseurl, conf.get_category()) + conf.set_categories([inp]) + baseurl = "%s%s/" % (baseurl, conf.get_categories()[0]) print wzrd_msg t = downloader.get_thread_links(baseurl) @@ -75,102 +75,109 @@ def walk_with_wizard(baseurl): print "Failed: ", failed print "Total: ", total -parser.set_usage( -"""%prog [options] +def parse_commands(): + conf = config.Configuration() + parser.set_usage( + """%prog [options] 4grab Copyright (C) 2009-2010 ryuslash This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you are welcome to redistribute it under certain conditions.""") -parser.add_option("-e", - nargs=2, - dest="confval", - metavar="CONF VALUE", - help="Set configuration option CONF to be VALUE") -parser.add_option("-c", - "--category", - dest="tempcat", - metavar="CATEGORY", - help="Set the category to CATEGORY only for this run") -parser.add_option("-t", - "--thread", - dest="thread", - metavar="THREAD", - help="Download only THREAD. If THREAD is only an ID, " - "CATEGORY must also be set. Otherwise, no problem :-)") -parser.add_option("-w", - "--wizard", - action="store_true", - dest="wizard", - help="I'll put on my robe and wizard hat and help you " - "get some of those pictures you like") -(options, args) = parser.parse_args() + parser.add_option("-e", + nargs=2, + dest="confval", + metavar="CONF VALUE", + help="Set configuration option CONF to be VALUE") + parser.add_option("-c", + "--category", + dest="tempcat", + metavar="CATEGORY", + help="Set the category to CATEGORY only for this run") + parser.add_option("-t", + "--thread", + dest="thread", + metavar="THREAD", + help="Download only THREAD. If THREAD is only an ID, " + "CATEGORY must also be set. Otherwise, no problem :-)") + parser.add_option("-w", + "--wizard", + action="store_true", + dest="wizard", + help="I'll put on my robe and wizard hat and help you " + "get some of those pictures you like") + (options, args) = parser.parse_args() -if options.confval and (options.tempcat - or options.thread - or options.wizard): - print "Can't configure something and do something else too." - exit(1) -if options.wizard and (options.tempcat - or options.thread - or options.confval): - print "Can't take a walk with the wizard and do something else too." - exit(1) - -if options.confval: - if not conf.option_exists(options.confval[0]): - print ("%s: error: %s is not a " - "valid configuration option") % (sys.argv[0], - options.confval[0]) + if options.confval and (options.tempcat + or options.thread + or options.wizard): + print "Can't configure something and do something else too." exit(1) - print "Setting", options.confval[0], "to", options.confval[1] - conf.set_option(options.confval[0], - options.confval[1]) - conf.save() - exit(0) -elif options.wizard: - try: - walk_with_wizard(base_url) - except KeyboardInterrupt: - print - print "Alright, no more wizard hat and robe then. Goodbye" - exit(0) + if options.wizard and (options.tempcat + or options.thread + or options.confval): + print "Can't take a walk with the wizard and do something else too." + exit(1) -elif options.thread: - try: - if options.thread[:7] == "http://": - t = downloader.get_image_links("", [options.thread]) - elif options.tempcat: - url = "%s%s/res/" % (base_url, options.tempcat) - t = downloader.get_image_links(url, [options.thread]) - else: - print ("if THREAD is not an absolute URL, " - "CATEGORY must also be specified") + if options.confval: + if not conf.option_exists(options.confval[0]): + print ("%s: error: %s is not a valid configuration option" + % (sys.argv[0], options.confval[0])) exit(1) - (skipped, failed, downloaded, total) = downloader.get_images(t) - print "Downloaded: ", downloaded - print "Skipped: ", skipped - print "Failed: ", failed - print "Total: ", total - except KeyboardInterrupt: - print - print "Goodbye" - exit(0) + print "Setting", options.confval[0], "to", options.confval[1] + conf.set_option(options.confval[0], + options.confval[1]) + conf.save() + exit(0) -elif options.tempcat: - conf.set_category(options.tempcat) + elif options.wizard: + try: + walk_with_wizard(base_url) + except KeyboardInterrupt: + print + print "Alright, no more wizard hat and robe then. Goodbye" + exit(0) -base_url = "%s%s/" % (base_url, conf.get_category()) + elif options.thread: + try: + if options.thread[:7] == "http://": + t = downloader.get_image_links("", [options.thread]) + elif options.tempcat: + url = "%s%s/res/" % (base_url, options.tempcat) + t = downloader.get_image_links(url, [options.thread]) + else: + print ("if THREAD is not an absolute URL, " + "CATEGORY must also be specified") + exit(1) + (skipped, failed, downloaded, total) = downloader.get_images(t) + print "Downloaded: ", downloaded + print "Skipped: ", skipped + print "Failed: ", failed + print "Total: ", total + except KeyboardInterrupt: + print + print "Goodbye" + exit(0) -try: - t = downloader.get_thread_links(base_url) - t = downloader.get_image_links(base_url, t) - (skipped, failed, downloaded, total) = downloader.get_images(t) - print "Downloaded: ", downloaded - print "Skipped: ", skipped - print "Failed: ", failed - print "Total: ", total -except KeyboardInterrupt: - print - print "So you don't want these images? Fine! I'll stop then." + elif options.tempcat: + conf.set_categories([options.tempcat]) + +#base_url = "%s%s/" % (base_url, conf.get_categories()) + +if __name__ == "__main__": + conf = config.Configuration() + parse_commands() + for category in conf.get_categories(): + base_url = "%s%s/" % (base_url, category) + try: + t = downloader.get_thread_links(base_url) + t = downloader.get_image_links(base_url, t) + (skipped, failed, downloaded, total) = downloader.get_images(t) + print "Downloaded: ", downloaded + print "Skipped: ", skipped + print "Failed: ", failed + print "Total: ", total + except KeyboardInterrupt: + print + print "So you don't want these images? Fine! I'll stop then." diff --git a/config.py b/config.py index 2f15933..66f3297 100644 --- a/config.py +++ b/config.py @@ -21,21 +21,19 @@ import os import ConfigParser import sys +# Get our reference point. preferably $HOME. homedir = os.getenv("HOME") if homedir is None: homedir = os.path.dirname(sys.argv[0]) class _Configuration(object): - def __init__(self): + def __init__(self, optioncreator): self.filename = os.path.join(os.path.join(homedir, ".4grab"), "config.cfg") self.configparser = ConfigParser.RawConfigParser() - - self.check() + self.optioncreator = optioncreator self.configparser.read(self.filename) - - def set_optioncreator(self, optioncreator): - self.optioncreator = optioncreator + print "__init__" def check(self): changed = False @@ -55,6 +53,17 @@ class _Configuration(object): "you would like the " "downloads to go: ") changed = True + # locations/archive + if not self.configparser.has_option("locations", "archive"): + self.create_option("locations", + "archive", + os.path.join(self.configparser.get("locations", + "download_base"), + ".arch"), + "Please enter where in {download_base} you " + "would like to store archived images (used for " + "checking what to download): ") + changed = True # settings if not self.configparser.has_section("settings"): self.configparser.add_section("settings") @@ -67,7 +76,15 @@ class _Configuration(object): "category you would like " "to download from: ") changed = True - + # settings/resolutions + if not self.configparser.has_option("settings", "resolutions"): + self.create_option("settings", + "resolutions", + "1600x1050,1900x1200,1900x1080", + "Please enter your preferred " + "resolutions (* for all)") + changed = True + # save if changed: self.save() @@ -79,12 +96,23 @@ class _Configuration(object): def get_download_location(self): return self.configparser.get("locations", "download_base") + def set_download_location(self, value): + self.configparser.set("locations", "download_base", value) - def get_category(self): - return self.configparser.get("settings", "categories") + def get_archive_location(self): + return self.configparser.get("locations", "archive") + def set_archive_location(self, value): + self.configparser.set("locations", "archive", value) - def set_category(self, value): - self.configparser.set("settings", "category", value) + def get_categories(self): + return self.configparser.get("settings", "categories").split(',') + def set_categories(self, value = []): + self.configparser.set("settings", "category", ','.join(value)) + + def get_resolutions(self): + return self.configparser.get("settings", "resolutions").split(',') + def set_resolutions(self, value = []): + self.configparser.set("settings", "resolutions", ','.join(value)) def option_exists(self, option): sections = self.configparser.sections() @@ -93,6 +121,7 @@ class _Configuration(object): return True return False + # Should only be used by the command-line def set_option(self, option, value): sec = None sections = self.configparser.sections() @@ -113,5 +142,15 @@ class _Configuration(object): configfile = open(self.filename, "w") self.configparser.write(configfile) -_configuration = _Configuration() -def Configuration(): return _configuration +_configuration = None +_optioncreator = None +def Configuration(): + global _optioncreator + global _configuration + + if _optioncreator is None: + raise ValueError("optioncreator must be set") + if _configuration is None: + _configuration = _Configuration(_optioncreator) + _configuration.check() + return _configuration diff --git a/download.py b/download.py index 05a5328..378d281 100644 --- a/download.py +++ b/download.py @@ -25,10 +25,12 @@ import htmlparser #import progressbar import config -conf = config.Configuration() -savedir = conf.get_download_location() -if not os.path.exists(savedir): - os.makedirs(savedir) +def get_savedir(): + conf = config.Configuration() + savedir = conf.get_download_location() + if not os.path.exists(savedir): + os.makedirs(savedir) + return savedir class Downloader(object): def __init__(self, progress_reporter): @@ -106,7 +108,7 @@ class Downloader(object): i = 1 for link in t: progress.show_progress(i) - filename = os.path.join(savedir, os.path.split(link)[1]) + filename = os.path.join(get_savedir(), os.path.split(link)[1]) if not os.path.exists(filename): tries = 10 while tries > 0: @@ -127,10 +129,4 @@ class Downloader(object): return (skipped, failed, downloaded, total) if __name__ == "__main__": - # Get a file-like object for the 4chan.org w/imgboard - base_url = "http://boards.4chan.org/" + conf.get_category() + "/" - - # Get the hyperlinks. - t = get_thread_links(base_url) - t = get_image_links(base_url, t) - get_images(t) + print "Don't run me, run 4grab.py"