diff options
author | Tom Willemse | 2013-05-09 20:07:49 +0200 |
---|---|---|
committer | Tom Willemse | 2013-05-09 20:07:49 +0200 |
commit | 15c5869b54e9a770b9abbeab1e990b713cdb1b3b (patch) | |
tree | 4aa805a44e8da93f4b0698520f16096d216d8ec3 /ryuslash/aggregator/management/commands | |
parent | 5f7df695ac6dd9ccc5444a8fc20fa8d97196953e (diff) | |
download | ryuslash.org-15c5869b54e9a770b9abbeab1e990b713cdb1b3b.tar.gz ryuslash.org-15c5869b54e9a770b9abbeab1e990b713cdb1b3b.zip |
Update for django 1.4
Diffstat (limited to 'ryuslash/aggregator/management/commands')
-rw-r--r-- | ryuslash/aggregator/management/commands/__init__.py | 0 | ||||
-rw-r--r-- | ryuslash/aggregator/management/commands/loadfeeds.py | 101 |
2 files changed, 101 insertions, 0 deletions
diff --git a/ryuslash/aggregator/management/commands/__init__.py b/ryuslash/aggregator/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/ryuslash/aggregator/management/commands/__init__.py diff --git a/ryuslash/aggregator/management/commands/loadfeeds.py b/ryuslash/aggregator/management/commands/loadfeeds.py new file mode 100644 index 0000000..f826312 --- /dev/null +++ b/ryuslash/aggregator/management/commands/loadfeeds.py @@ -0,0 +1,101 @@ +import feedparser +import datetime +import markdown +import re +import os +import urllib2 + +from django.core.management.base import BaseCommand + +from aggregator.models import Post +import settings + +class Command(BaseCommand): + help = "Load data from saved feeds." + + def prep_feedname(self, value): + value = re.sub('[^\w\s-]', '', value).strip().lower() + return re.sub('[-\s]+', '-', value) + + def get_ext(self, options): + if 'favicon_ext' in options.keys(): + return options['favicon_ext'] + else: + return 'ico' + + def get_logopath(self, feedname, options): + ext = self.get_ext(options) + filename = self.prep_feedname(feedname) + '.' + ext + basedir = os.path.dirname(os.path.abspath(settings.__file__)) + return os.path.join(basedir, 'static/images/logos', filename) + + def have_logo(self, feedname, options): + logopath = self.get_logopath(feedname, options) + return os.path.exists(logopath) + + def save_logo(self, feedname, options): + ext = self.get_ext(options) + url = options['base_url'] + '/favicon.' + ext + + try: + logo = urllib2.urlopen(url) + except: + return + + save = open(self.get_logopath(feedname, options), 'w') + + save.write(logo.read()) + save.close() + logo.close() + + def construct_feed_url(self, feed): + return feed['base_url'] + feed['feed_url'] + + def handle(self, *args, **kwargs): + for feedname, feedoptions in settings.FEEDS.iteritems(): + parsed = \ + feedparser.parse(self.construct_feed_url(feedoptions)) + icon = self.prep_feedname(feedname) + '.' \ + + self.get_ext(feedoptions) + newcount = 0 + + if not self.have_logo(feedname, feedoptions): + self.save_logo(feedname, feedoptions) + + for entry in parsed.entries: + if Post.objects.filter(post_id=entry.id).exists(): + continue + + dt = entry.updated_parsed \ + or entry.published_parsed + + if dt: + updated = datetime.datetime( + dt.tm_year, dt.tm_mon, dt.tm_mday, + dt.tm_hour, dt.tm_min, dt.tm_sec) + else: + updated = datetime.datetime.now() + + if 'content' in entry.keys(): + content = entry.content[0]['value'] + else: + content = entry.summary + + if feedoptions['markdown']: + content = markdown.markdown(content) + + if feedoptions.get('nl2br'): + content = re.sub('\n', '</br>\n', content) + + post = Post(post_id=entry.id, + title=entry.title, + category=feedoptions['category'], + link=entry.link, + updated=updated, + icon=icon, + content=content) + + post.save() + newcount += 1 + + print 'Grabbed %d new feeds from %s' % (newcount, feedname) |