1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
import feedparser
import datetime
import markdown
import re
import os
import urllib2
from django.core.management.base import BaseCommand
from aggregator.models import Post
import settings
class Command(BaseCommand):
help = "Load data from saved feeds."
def prep_feedname(self, value):
value = re.sub('[^\w\s-]', '', value).strip().lower()
return re.sub('[-\s]+', '-', value)
def get_ext(self, options):
if 'favicon_ext' in options.keys():
return options['favicon_ext']
else:
return 'ico'
def get_logopath(self, feedname, options):
ext = self.get_ext(options)
filename = self.prep_feedname(feedname) + '.' + ext
basedir = os.path.dirname(os.path.abspath(settings.__file__))
return os.path.join(basedir, 'static/images/logos', filename)
def have_logo(self, feedname, options):
logopath = self.get_logopath(feedname, options)
return os.path.exists(logopath)
def save_logo(self, feedname, options):
ext = self.get_ext(options)
url = options['base_url'] + '/favicon.' + ext
try:
logo = urllib2.urlopen(url)
except:
return
save = open(self.get_logopath(feedname, options), 'w')
save.write(logo.read())
save.close()
logo.close()
def construct_feed_url(self, feed):
return feed['base_url'] + feed['feed_url']
def handle(self, *args, **kwargs):
for feedname, feedoptions in settings.FEEDS.iteritems():
parsed = \
feedparser.parse(self.construct_feed_url(feedoptions))
icon = self.prep_feedname(feedname) + '.' \
+ self.get_ext(feedoptions)
newcount = 0
if not self.have_logo(feedname, feedoptions):
self.save_logo(feedname, feedoptions)
for entry in parsed.entries:
if Post.objects.filter(post_id=entry.id).exists():
continue
dt = entry.updated_parsed \
or entry.published_parsed
if dt:
updated = datetime.datetime(
dt.tm_year, dt.tm_mon, dt.tm_mday,
dt.tm_hour, dt.tm_min, dt.tm_sec)
else:
updated = datetime.datetime.now()
if 'content' in entry.keys():
content = entry.content[0]['value']
else:
content = entry.summary
if feedoptions['markdown']:
content = markdown.markdown(content)
if feedoptions.get('nl2br'):
content = re.sub('\n', '</br>\n', content)
post = Post(
post_id=entry.id,
title=entry.title if feedoptions.get('title') else '',
category=feedoptions['category'],
link=entry.link,
updated=updated,
icon=icon,
content=content
)
post.save()
newcount += 1
print 'Grabbed %d new feeds from %s' % (newcount, feedname)
|
