optioncreator, extra options
config now has an optioncreator property that will allow another module to select how to prompt for a property resolutions and archive options have been added to config file for sorting functionality
This commit is contained in:
ryuslash
parent
8e101c92f9
commit
4a9cc7e2b6
3 changed files with 158 additions and 116 deletions
189
4grab.py
189
4grab.py
|
|
@ -23,6 +23,7 @@ import optparse
|
|||
import sys
|
||||
|
||||
import config
|
||||
|
||||
import download
|
||||
import progressbar
|
||||
|
||||
|
|
@ -31,15 +32,14 @@ def raw_input_with_default(default, prompt):
|
|||
if inp == "":
|
||||
return default
|
||||
return inp
|
||||
config._optioncreator = raw_input_with_default
|
||||
|
||||
base_url = "http://boards.4chan.org/"
|
||||
parser = optparse.OptionParser()
|
||||
downloader = download.Downloader(progressbar.Progress)
|
||||
conf = config.Configuration()
|
||||
|
||||
conf.set_optioncreator(raw_input_with_default)
|
||||
|
||||
def walk_with_wizard(baseurl):
|
||||
conf = config.Configuration()
|
||||
wzrd_msg = "Pilates! *SHAZAM* Here they come!"
|
||||
print "Alright, let me put on my robe and wizard hat."
|
||||
|
||||
|
|
@ -63,8 +63,8 @@ def walk_with_wizard(baseurl):
|
|||
[thread])
|
||||
else:
|
||||
inp = raw_input("Which category would you like to download? ")
|
||||
conf.set_category(inp)
|
||||
baseurl = "%s%s/" % (baseurl, conf.get_category())
|
||||
conf.set_categories([inp])
|
||||
baseurl = "%s%s/" % (baseurl, conf.get_categories()[0])
|
||||
|
||||
print wzrd_msg
|
||||
t = downloader.get_thread_links(baseurl)
|
||||
|
|
@ -75,102 +75,109 @@ def walk_with_wizard(baseurl):
|
|||
print "Failed: ", failed
|
||||
print "Total: ", total
|
||||
|
||||
parser.set_usage(
|
||||
"""%prog [options]
|
||||
def parse_commands():
|
||||
conf = config.Configuration()
|
||||
parser.set_usage(
|
||||
"""%prog [options]
|
||||
|
||||
4grab Copyright (C) 2009-2010 ryuslash
|
||||
This program comes with ABSOLUTELY NO WARRANTY.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions.""")
|
||||
parser.add_option("-e",
|
||||
nargs=2,
|
||||
dest="confval",
|
||||
metavar="CONF VALUE",
|
||||
help="Set configuration option CONF to be VALUE")
|
||||
parser.add_option("-c",
|
||||
"--category",
|
||||
dest="tempcat",
|
||||
metavar="CATEGORY",
|
||||
help="Set the category to CATEGORY only for this run")
|
||||
parser.add_option("-t",
|
||||
"--thread",
|
||||
dest="thread",
|
||||
metavar="THREAD",
|
||||
help="Download only THREAD. If THREAD is only an ID, "
|
||||
"CATEGORY must also be set. Otherwise, no problem :-)")
|
||||
parser.add_option("-w",
|
||||
"--wizard",
|
||||
action="store_true",
|
||||
dest="wizard",
|
||||
help="I'll put on my robe and wizard hat and help you "
|
||||
"get some of those pictures you like")
|
||||
(options, args) = parser.parse_args()
|
||||
parser.add_option("-e",
|
||||
nargs=2,
|
||||
dest="confval",
|
||||
metavar="CONF VALUE",
|
||||
help="Set configuration option CONF to be VALUE")
|
||||
parser.add_option("-c",
|
||||
"--category",
|
||||
dest="tempcat",
|
||||
metavar="CATEGORY",
|
||||
help="Set the category to CATEGORY only for this run")
|
||||
parser.add_option("-t",
|
||||
"--thread",
|
||||
dest="thread",
|
||||
metavar="THREAD",
|
||||
help="Download only THREAD. If THREAD is only an ID, "
|
||||
"CATEGORY must also be set. Otherwise, no problem :-)")
|
||||
parser.add_option("-w",
|
||||
"--wizard",
|
||||
action="store_true",
|
||||
dest="wizard",
|
||||
help="I'll put on my robe and wizard hat and help you "
|
||||
"get some of those pictures you like")
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if options.confval and (options.tempcat
|
||||
or options.thread
|
||||
or options.wizard):
|
||||
print "Can't configure something and do something else too."
|
||||
exit(1)
|
||||
if options.wizard and (options.tempcat
|
||||
or options.thread
|
||||
or options.confval):
|
||||
print "Can't take a walk with the wizard and do something else too."
|
||||
exit(1)
|
||||
|
||||
if options.confval:
|
||||
if not conf.option_exists(options.confval[0]):
|
||||
print ("%s: error: %s is not a "
|
||||
"valid configuration option") % (sys.argv[0],
|
||||
options.confval[0])
|
||||
if options.confval and (options.tempcat
|
||||
or options.thread
|
||||
or options.wizard):
|
||||
print "Can't configure something and do something else too."
|
||||
exit(1)
|
||||
print "Setting", options.confval[0], "to", options.confval[1]
|
||||
conf.set_option(options.confval[0],
|
||||
options.confval[1])
|
||||
conf.save()
|
||||
exit(0)
|
||||
|
||||
elif options.wizard:
|
||||
try:
|
||||
walk_with_wizard(base_url)
|
||||
except KeyboardInterrupt:
|
||||
print
|
||||
print "Alright, no more wizard hat and robe then. Goodbye"
|
||||
exit(0)
|
||||
if options.wizard and (options.tempcat
|
||||
or options.thread
|
||||
or options.confval):
|
||||
print "Can't take a walk with the wizard and do something else too."
|
||||
exit(1)
|
||||
|
||||
elif options.thread:
|
||||
try:
|
||||
if options.thread[:7] == "http://":
|
||||
t = downloader.get_image_links("", [options.thread])
|
||||
elif options.tempcat:
|
||||
url = "%s%s/res/" % (base_url, options.tempcat)
|
||||
t = downloader.get_image_links(url, [options.thread])
|
||||
else:
|
||||
print ("if THREAD is not an absolute URL, "
|
||||
"CATEGORY must also be specified")
|
||||
if options.confval:
|
||||
if not conf.option_exists(options.confval[0]):
|
||||
print ("%s: error: %s is not a valid configuration option"
|
||||
% (sys.argv[0], options.confval[0]))
|
||||
exit(1)
|
||||
(skipped, failed, downloaded, total) = downloader.get_images(t)
|
||||
print "Downloaded: ", downloaded
|
||||
print "Skipped: ", skipped
|
||||
print "Failed: ", failed
|
||||
print "Total: ", total
|
||||
except KeyboardInterrupt:
|
||||
print
|
||||
print "Goodbye"
|
||||
exit(0)
|
||||
print "Setting", options.confval[0], "to", options.confval[1]
|
||||
conf.set_option(options.confval[0],
|
||||
options.confval[1])
|
||||
conf.save()
|
||||
exit(0)
|
||||
|
||||
elif options.tempcat:
|
||||
conf.set_category(options.tempcat)
|
||||
elif options.wizard:
|
||||
try:
|
||||
walk_with_wizard(base_url)
|
||||
except KeyboardInterrupt:
|
||||
print
|
||||
print "Alright, no more wizard hat and robe then. Goodbye"
|
||||
exit(0)
|
||||
|
||||
base_url = "%s%s/" % (base_url, conf.get_category())
|
||||
elif options.thread:
|
||||
try:
|
||||
if options.thread[:7] == "http://":
|
||||
t = downloader.get_image_links("", [options.thread])
|
||||
elif options.tempcat:
|
||||
url = "%s%s/res/" % (base_url, options.tempcat)
|
||||
t = downloader.get_image_links(url, [options.thread])
|
||||
else:
|
||||
print ("if THREAD is not an absolute URL, "
|
||||
"CATEGORY must also be specified")
|
||||
exit(1)
|
||||
(skipped, failed, downloaded, total) = downloader.get_images(t)
|
||||
print "Downloaded: ", downloaded
|
||||
print "Skipped: ", skipped
|
||||
print "Failed: ", failed
|
||||
print "Total: ", total
|
||||
except KeyboardInterrupt:
|
||||
print
|
||||
print "Goodbye"
|
||||
exit(0)
|
||||
|
||||
try:
|
||||
t = downloader.get_thread_links(base_url)
|
||||
t = downloader.get_image_links(base_url, t)
|
||||
(skipped, failed, downloaded, total) = downloader.get_images(t)
|
||||
print "Downloaded: ", downloaded
|
||||
print "Skipped: ", skipped
|
||||
print "Failed: ", failed
|
||||
print "Total: ", total
|
||||
except KeyboardInterrupt:
|
||||
print
|
||||
print "So you don't want these images? Fine! I'll stop then."
|
||||
elif options.tempcat:
|
||||
conf.set_categories([options.tempcat])
|
||||
|
||||
#base_url = "%s%s/" % (base_url, conf.get_categories())
|
||||
|
||||
if __name__ == "__main__":
|
||||
conf = config.Configuration()
|
||||
parse_commands()
|
||||
for category in conf.get_categories():
|
||||
base_url = "%s%s/" % (base_url, category)
|
||||
try:
|
||||
t = downloader.get_thread_links(base_url)
|
||||
t = downloader.get_image_links(base_url, t)
|
||||
(skipped, failed, downloaded, total) = downloader.get_images(t)
|
||||
print "Downloaded: ", downloaded
|
||||
print "Skipped: ", skipped
|
||||
print "Failed: ", failed
|
||||
print "Total: ", total
|
||||
except KeyboardInterrupt:
|
||||
print
|
||||
print "So you don't want these images? Fine! I'll stop then."
|
||||
|
|
|
|||
65
config.py
65
config.py
|
|
@ -21,21 +21,19 @@ import os
|
|||
import ConfigParser
|
||||
import sys
|
||||
|
||||
# Get our reference point. preferably $HOME.
|
||||
homedir = os.getenv("HOME")
|
||||
if homedir is None:
|
||||
homedir = os.path.dirname(sys.argv[0])
|
||||
|
||||
class _Configuration(object):
|
||||
def __init__(self):
|
||||
def __init__(self, optioncreator):
|
||||
self.filename = os.path.join(os.path.join(homedir, ".4grab"), "config.cfg")
|
||||
self.configparser = ConfigParser.RawConfigParser()
|
||||
|
||||
self.check()
|
||||
self.optioncreator = optioncreator
|
||||
|
||||
self.configparser.read(self.filename)
|
||||
|
||||
def set_optioncreator(self, optioncreator):
|
||||
self.optioncreator = optioncreator
|
||||
print "__init__"
|
||||
|
||||
def check(self):
|
||||
changed = False
|
||||
|
|
@ -55,6 +53,17 @@ class _Configuration(object):
|
|||
"you would like the "
|
||||
"downloads to go: ")
|
||||
changed = True
|
||||
# locations/archive
|
||||
if not self.configparser.has_option("locations", "archive"):
|
||||
self.create_option("locations",
|
||||
"archive",
|
||||
os.path.join(self.configparser.get("locations",
|
||||
"download_base"),
|
||||
".arch"),
|
||||
"Please enter where in {download_base} you "
|
||||
"would like to store archived images (used for "
|
||||
"checking what to download): ")
|
||||
changed = True
|
||||
# settings
|
||||
if not self.configparser.has_section("settings"):
|
||||
self.configparser.add_section("settings")
|
||||
|
|
@ -67,7 +76,15 @@ class _Configuration(object):
|
|||
"category you would like "
|
||||
"to download from: ")
|
||||
changed = True
|
||||
|
||||
# settings/resolutions
|
||||
if not self.configparser.has_option("settings", "resolutions"):
|
||||
self.create_option("settings",
|
||||
"resolutions",
|
||||
"1600x1050,1900x1200,1900x1080",
|
||||
"Please enter your preferred "
|
||||
"resolutions (* for all)")
|
||||
changed = True
|
||||
# save
|
||||
if changed:
|
||||
self.save()
|
||||
|
||||
|
|
@ -79,12 +96,23 @@ class _Configuration(object):
|
|||
|
||||
def get_download_location(self):
|
||||
return self.configparser.get("locations", "download_base")
|
||||
def set_download_location(self, value):
|
||||
self.configparser.set("locations", "download_base", value)
|
||||
|
||||
def get_category(self):
|
||||
return self.configparser.get("settings", "categories")
|
||||
def get_archive_location(self):
|
||||
return self.configparser.get("locations", "archive")
|
||||
def set_archive_location(self, value):
|
||||
self.configparser.set("locations", "archive", value)
|
||||
|
||||
def set_category(self, value):
|
||||
self.configparser.set("settings", "category", value)
|
||||
def get_categories(self):
|
||||
return self.configparser.get("settings", "categories").split(',')
|
||||
def set_categories(self, value = []):
|
||||
self.configparser.set("settings", "category", ','.join(value))
|
||||
|
||||
def get_resolutions(self):
|
||||
return self.configparser.get("settings", "resolutions").split(',')
|
||||
def set_resolutions(self, value = []):
|
||||
self.configparser.set("settings", "resolutions", ','.join(value))
|
||||
|
||||
def option_exists(self, option):
|
||||
sections = self.configparser.sections()
|
||||
|
|
@ -93,6 +121,7 @@ class _Configuration(object):
|
|||
return True
|
||||
return False
|
||||
|
||||
# Should only be used by the command-line
|
||||
def set_option(self, option, value):
|
||||
sec = None
|
||||
sections = self.configparser.sections()
|
||||
|
|
@ -113,5 +142,15 @@ class _Configuration(object):
|
|||
configfile = open(self.filename, "w")
|
||||
self.configparser.write(configfile)
|
||||
|
||||
_configuration = _Configuration()
|
||||
def Configuration(): return _configuration
|
||||
_configuration = None
|
||||
_optioncreator = None
|
||||
def Configuration():
|
||||
global _optioncreator
|
||||
global _configuration
|
||||
|
||||
if _optioncreator is None:
|
||||
raise ValueError("optioncreator must be set")
|
||||
if _configuration is None:
|
||||
_configuration = _Configuration(_optioncreator)
|
||||
_configuration.check()
|
||||
return _configuration
|
||||
|
|
|
|||
20
download.py
20
download.py
|
|
@ -25,10 +25,12 @@ import htmlparser
|
|||
#import progressbar
|
||||
import config
|
||||
|
||||
conf = config.Configuration()
|
||||
savedir = conf.get_download_location()
|
||||
if not os.path.exists(savedir):
|
||||
os.makedirs(savedir)
|
||||
def get_savedir():
|
||||
conf = config.Configuration()
|
||||
savedir = conf.get_download_location()
|
||||
if not os.path.exists(savedir):
|
||||
os.makedirs(savedir)
|
||||
return savedir
|
||||
|
||||
class Downloader(object):
|
||||
def __init__(self, progress_reporter):
|
||||
|
|
@ -106,7 +108,7 @@ class Downloader(object):
|
|||
i = 1
|
||||
for link in t:
|
||||
progress.show_progress(i)
|
||||
filename = os.path.join(savedir, os.path.split(link)[1])
|
||||
filename = os.path.join(get_savedir(), os.path.split(link)[1])
|
||||
if not os.path.exists(filename):
|
||||
tries = 10
|
||||
while tries > 0:
|
||||
|
|
@ -127,10 +129,4 @@ class Downloader(object):
|
|||
return (skipped, failed, downloaded, total)
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Get a file-like object for the 4chan.org w/imgboard
|
||||
base_url = "http://boards.4chan.org/" + conf.get_category() + "/"
|
||||
|
||||
# Get the hyperlinks.
|
||||
t = get_thread_links(base_url)
|
||||
t = get_image_links(base_url, t)
|
||||
get_images(t)
|
||||
print "Don't run me, run 4grab.py"
|
||||
|
|
|
|||
Loading…
Reference in a new issue