From 2db5555609b4ab157ea836f9611c6faa80508cf3 Mon Sep 17 00:00:00 2001 From: ryuslash Date: Thu, 8 Apr 2010 15:14:12 +0200 Subject: Archive lookup 4grab was saving the archived images to the wrong place, which is now fixed --- util.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 util.py (limited to 'util.py') diff --git a/util.py b/util.py new file mode 100644 index 0000000..9e8214e --- /dev/null +++ b/util.py @@ -0,0 +1,7 @@ +import os +import sys + +homedir = os.getenv("HOME") +if homedir is None: + homedir = os.path.dirname(sys.argv[0]) +confdir = os.path.join(homedir, ".4grab") -- cgit v1.2.3-54-g00ecf From af529bcd4e214c13b7810eeba4d425c4a1129b58 Mon Sep 17 00:00:00 2001 From: ryuslash Date: Mon, 19 Apr 2010 09:16:04 +0200 Subject: Not checking and kirbybase Using kirbybase to store downloaded images A nasty bug that always returned the wrong result when checking whether an image had already been downloaded has been fixed --- .gitignore | 3 ++- 4grab.py | 6 +----- backend.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ download.py | 4 ++++ sorter.py | 24 ++++++++---------------- util.py | 6 ++++++ 6 files changed, 81 insertions(+), 22 deletions(-) create mode 100644 backend.py (limited to 'util.py') diff --git a/.gitignore b/.gitignore index 58ca1a8..2dd1b6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *~ *.pyc \#*\# -.* \ No newline at end of file +.* +modules/plane.tbl \ No newline at end of file diff --git a/4grab.py b/4grab.py index 4dfca95..367cc04 100755 --- a/4grab.py +++ b/4grab.py @@ -24,16 +24,12 @@ import sys import os import config +from util import raw_input_with_default import download import progressbar import sorter -def raw_input_with_default(default, prompt): - inp = raw_input("%s (default=%s): " % (prompt, default)) - if inp == "": - return default - return inp config._optioncreator = raw_input_with_default base_url = "http://boards.4chan.org/" diff --git a/backend.py b/backend.py new file mode 100644 index 0000000..8dffb3e --- /dev/null +++ b/backend.py @@ -0,0 +1,60 @@ +import os +import modules.kirbybase +from util import confdir,raw_input_with_default +import config + +class _Backend(object): + """ A class that communicates with the datastore """ + def __init__(self): + self.table = os.path.join(confdir, "images.tbl") + self.store = modules.kirbybase.KirbyBase() + self.__collection = "" + self.__new_collection = [] + + self.load() + + def create_store_if_needed(self): + if not os.path.exists(self.table): + return self.store.create(self.table, ["filename:String"]) + return True + + def add(self, filename): + self.__new_collection.append(filename) + + def check(self, filename): + collected = filename in self.__collection + downloaded = filename in self.__new_collection + + if not downloaded: + self.add(filename) + + if collected or downloaded: + return True + + return False + + def save(self): + os.remove(self.table) + self.create_store_if_needed() + for f in self.__new_collection: + self.store.insert(self.table, [f]) + + def load(self): + if os.path.exists(self.table): + self.__collection = self.store.select(self.table, ['recno'], ['*'], ['filename'], returnType="report") + +_backend = None +def Backend(): + global _backend + + if _backend == None: + _backend = _Backend() + return _backend + +if __name__ == "__main__": + backend = Backend() + config._optioncreator = raw_input_with_default + cfg = config.Configuration() + + for f in os.listdir(cfg.get_archive_location()): + backend.add(f) diff --git a/download.py b/download.py index 68315bc..42fcd52 100644 --- a/download.py +++ b/download.py @@ -144,6 +144,10 @@ class Downloader(object): i += 1 progress.complete() + + be = backend.Backend() + be.save() + return (skipped, failed, downloaded, total) if __name__ == "__main__": diff --git a/sorter.py b/sorter.py index 4022b55..5cb35e9 100644 --- a/sorter.py +++ b/sorter.py @@ -22,6 +22,7 @@ import Image import shutil import os import datetime +import backend def dummy_option_creator(value1, value2): pass config._optioncreator = dummy_option_creator @@ -36,7 +37,7 @@ class Sorter: download_base = self.conf.get_download_location() retval = True - if self.check_filename(filename): + if not self.check_filename(filename): image = None try: image = Image.open(os.path.join(download_base, @@ -59,10 +60,9 @@ class Sorter: self.copy(filename, folderpath) break - self.archive(filename) - self.remove(filename) - return retval + self.remove(filename) + return retval def copy(self, filename, destpath): download_base = self.conf.get_download_location() @@ -84,20 +84,12 @@ class Sorter: print "\nHow can this even happen?! Copying", source, "to", dest def archive(self, filename): - location = self.conf.get_archive_location() - if not os.path.exists(location): - os.makedirs(location) - - dest = os.path.join(location, os.path.basename(filename)) - #print dest - f = open(dest, "w") - file.close(f) - + be = backend.Backend() + be.add(os.path.basename(filename)) def archive_check(self, filename): - archive_path = self.conf.get_archive_location() - fullname = os.path.join(archive_path, filename) - return os.path.exists(fullname) + be = backend.Backend() + return be.check(os.path.basename(filename)) def check_filename(self, filename): ext = os.path.splitext(filename)[1] diff --git a/util.py b/util.py index 9e8214e..69a24e5 100644 --- a/util.py +++ b/util.py @@ -5,3 +5,9 @@ homedir = os.getenv("HOME") if homedir is None: homedir = os.path.dirname(sys.argv[0]) confdir = os.path.join(homedir, ".4grab") + +def raw_input_with_default(default, prompt): + inp = raw_input("%s (default=%s): " % (prompt, default)) + if inp == "": + return default + return inp -- cgit v1.2.3-54-g00ecf