aboutsummaryrefslogtreecommitdiffstats
path: root/ryuslash/aggregator
diff options
context:
space:
mode:
Diffstat (limited to 'ryuslash/aggregator')
-rw-r--r--ryuslash/aggregator/README.org5
-rw-r--r--ryuslash/aggregator/__init__.py0
-rw-r--r--ryuslash/aggregator/feeds.py23
-rw-r--r--ryuslash/aggregator/management/__init__.py0
-rw-r--r--ryuslash/aggregator/management/commands/__init__.py0
-rw-r--r--ryuslash/aggregator/management/commands/loadfeeds.py101
-rw-r--r--ryuslash/aggregator/migrations/0001_initial.py40
-rw-r--r--ryuslash/aggregator/migrations/__init__.py0
-rw-r--r--ryuslash/aggregator/models.py14
-rw-r--r--ryuslash/aggregator/templates/aggregator/base.html46
-rw-r--r--ryuslash/aggregator/templates/aggregator/posts.html43
-rw-r--r--ryuslash/aggregator/templatetags/__init__.py0
-rw-r--r--ryuslash/aggregator/templatetags/posts_extras.py20
-rw-r--r--ryuslash/aggregator/views.py22
14 files changed, 314 insertions, 0 deletions
diff --git a/ryuslash/aggregator/README.org b/ryuslash/aggregator/README.org
new file mode 100644
index 0000000..1bca9e7
--- /dev/null
+++ b/ryuslash/aggregator/README.org
@@ -0,0 +1,5 @@
+* aggregator
+
+ A simple django app that collects posts from specified feeds. It
+ keeps track of which was last updated when and copies the data to
+ its own tables.
diff --git a/ryuslash/aggregator/__init__.py b/ryuslash/aggregator/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ryuslash/aggregator/__init__.py
diff --git a/ryuslash/aggregator/feeds.py b/ryuslash/aggregator/feeds.py
new file mode 100644
index 0000000..75e5d4f
--- /dev/null
+++ b/ryuslash/aggregator/feeds.py
@@ -0,0 +1,23 @@
+from django.contrib.syndication.views import Feed
+
+from .models import Post
+
+class LatestPostsFeed(Feed):
+ title = "ryuslash's RSS feed"
+ link = "/"
+ description = "Updates by ryuslash"
+
+ def items(self):
+ return Post.objects.all()[:20]
+
+ def item_title(self, item):
+ return item.title
+
+ def item_description(self, item):
+ return item.body
+
+ def item_link(self, item):
+ return "/post/%d/" % item.pk
+
+ def item_pubdate(self, item):
+ return item.updated
diff --git a/ryuslash/aggregator/management/__init__.py b/ryuslash/aggregator/management/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ryuslash/aggregator/management/__init__.py
diff --git a/ryuslash/aggregator/management/commands/__init__.py b/ryuslash/aggregator/management/commands/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ryuslash/aggregator/management/commands/__init__.py
diff --git a/ryuslash/aggregator/management/commands/loadfeeds.py b/ryuslash/aggregator/management/commands/loadfeeds.py
new file mode 100644
index 0000000..f826312
--- /dev/null
+++ b/ryuslash/aggregator/management/commands/loadfeeds.py
@@ -0,0 +1,101 @@
+import feedparser
+import datetime
+import markdown
+import re
+import os
+import urllib2
+
+from django.core.management.base import BaseCommand
+
+from aggregator.models import Post
+import settings
+
+class Command(BaseCommand):
+ help = "Load data from saved feeds."
+
+ def prep_feedname(self, value):
+ value = re.sub('[^\w\s-]', '', value).strip().lower()
+ return re.sub('[-\s]+', '-', value)
+
+ def get_ext(self, options):
+ if 'favicon_ext' in options.keys():
+ return options['favicon_ext']
+ else:
+ return 'ico'
+
+ def get_logopath(self, feedname, options):
+ ext = self.get_ext(options)
+ filename = self.prep_feedname(feedname) + '.' + ext
+ basedir = os.path.dirname(os.path.abspath(settings.__file__))
+ return os.path.join(basedir, 'static/images/logos', filename)
+
+ def have_logo(self, feedname, options):
+ logopath = self.get_logopath(feedname, options)
+ return os.path.exists(logopath)
+
+ def save_logo(self, feedname, options):
+ ext = self.get_ext(options)
+ url = options['base_url'] + '/favicon.' + ext
+
+ try:
+ logo = urllib2.urlopen(url)
+ except:
+ return
+
+ save = open(self.get_logopath(feedname, options), 'w')
+
+ save.write(logo.read())
+ save.close()
+ logo.close()
+
+ def construct_feed_url(self, feed):
+ return feed['base_url'] + feed['feed_url']
+
+ def handle(self, *args, **kwargs):
+ for feedname, feedoptions in settings.FEEDS.iteritems():
+ parsed = \
+ feedparser.parse(self.construct_feed_url(feedoptions))
+ icon = self.prep_feedname(feedname) + '.' \
+ + self.get_ext(feedoptions)
+ newcount = 0
+
+ if not self.have_logo(feedname, feedoptions):
+ self.save_logo(feedname, feedoptions)
+
+ for entry in parsed.entries:
+ if Post.objects.filter(post_id=entry.id).exists():
+ continue
+
+ dt = entry.updated_parsed \
+ or entry.published_parsed
+
+ if dt:
+ updated = datetime.datetime(
+ dt.tm_year, dt.tm_mon, dt.tm_mday,
+ dt.tm_hour, dt.tm_min, dt.tm_sec)
+ else:
+ updated = datetime.datetime.now()
+
+ if 'content' in entry.keys():
+ content = entry.content[0]['value']
+ else:
+ content = entry.summary
+
+ if feedoptions['markdown']:
+ content = markdown.markdown(content)
+
+ if feedoptions.get('nl2br'):
+ content = re.sub('\n', '</br>\n', content)
+
+ post = Post(post_id=entry.id,
+ title=entry.title,
+ category=feedoptions['category'],
+ link=entry.link,
+ updated=updated,
+ icon=icon,
+ content=content)
+
+ post.save()
+ newcount += 1
+
+ print 'Grabbed %d new feeds from %s' % (newcount, feedname)
diff --git a/ryuslash/aggregator/migrations/0001_initial.py b/ryuslash/aggregator/migrations/0001_initial.py
new file mode 100644
index 0000000..550a389
--- /dev/null
+++ b/ryuslash/aggregator/migrations/0001_initial.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ # Adding model 'Post'
+ db.create_table('aggregator_post', (
+ ('post_id', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255, primary_key=True)),
+ ('title', self.gf('django.db.models.fields.CharField')(max_length=500)),
+ ('category', self.gf('django.db.models.fields.CharField')(max_length=255)),
+ ('link', self.gf('django.db.models.fields.URLField')(max_length=255)),
+ ('updated', self.gf('django.db.models.fields.DateTimeField')()),
+ ('content', self.gf('django.db.models.fields.TextField')()),
+ ('icon', self.gf('django.db.models.fields.CharField')(max_length=255)),
+ ))
+ db.send_create_signal('aggregator', ['Post'])
+
+ def backwards(self, orm):
+ # Deleting model 'Post'
+ db.delete_table('aggregator_post')
+
+ models = {
+ 'aggregator.post': {
+ 'Meta': {'ordering': "['-updated']", 'object_name': 'Post'},
+ 'category': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'content': ('django.db.models.fields.TextField', [], {}),
+ 'icon': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ 'link': ('django.db.models.fields.URLField', [], {'max_length': '255'}),
+ 'post_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'primary_key': 'True'}),
+ 'title': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+ 'updated': ('django.db.models.fields.DateTimeField', [], {})
+ }
+ }
+
+ complete_apps = ['aggregator'] \ No newline at end of file
diff --git a/ryuslash/aggregator/migrations/__init__.py b/ryuslash/aggregator/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ryuslash/aggregator/migrations/__init__.py
diff --git a/ryuslash/aggregator/models.py b/ryuslash/aggregator/models.py
new file mode 100644
index 0000000..34fb5a3
--- /dev/null
+++ b/ryuslash/aggregator/models.py
@@ -0,0 +1,14 @@
+from django.db import models
+
+class Post(models.Model):
+ post_id = models.CharField(max_length=255, unique=True,
+ primary_key=True)
+ title = models.CharField(max_length=500)
+ category = models.CharField(max_length=255)
+ link = models.URLField(max_length=255)
+ updated = models.DateTimeField()
+ content = models.TextField()
+ icon = models.CharField(max_length=255)
+
+ class Meta:
+ ordering = [ '-updated' ]
diff --git a/ryuslash/aggregator/templates/aggregator/base.html b/ryuslash/aggregator/templates/aggregator/base.html
new file mode 100644
index 0000000..215ec7e
--- /dev/null
+++ b/ryuslash/aggregator/templates/aggregator/base.html
@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+
+<html lang="en">
+ <head>
+ <meta charset="utf-8" />
+ <title>ryuslash</title>
+ <link href="/static/main.css" rel="stylesheet" />
+ <link rel="shortcut icon" href="/static/favicon.png" />
+ {% block head %}{% endblock %}
+ </head>
+ <body>
+
+ <header>
+ <img src="/static/logo.png" alt="ryuslash.org" title="ryuslash.org"/>
+ <h1>
+ <a href="http://ryuslash.org">
+ <span id="logo-blue">ryu</span><span id="logo-orange">slash</span>
+ </a>
+ </h1>
+
+ <nav>
+ <ul>
+ {% if category == 'post' %}
+ <li class="selected top">
+ {% else %}
+ <li class="top">
+ {% endif %}
+ <a href="/">posts</a>
+ </li>
+
+ {% if category == 'activity' %}
+ <li class="selected bottom">
+ {% else %}
+ <li class="bottom">
+ {% endif %}
+ <a href="/activity/">activities</a>
+ </li>
+ </ul>
+ </nav>
+ </header>
+
+ <section class="content">
+ {% block content %}{% endblock %}
+ </div>
+ </body>
+</html>
diff --git a/ryuslash/aggregator/templates/aggregator/posts.html b/ryuslash/aggregator/templates/aggregator/posts.html
new file mode 100644
index 0000000..b70cb0d
--- /dev/null
+++ b/ryuslash/aggregator/templates/aggregator/posts.html
@@ -0,0 +1,43 @@
+{% extends "aggregator/base.html" %}
+{% load posts_extras %}
+
+{% block head %}
+<link href="/feeds/posts/" rel="alternate" type="application/rss+xml"
+ title="All posts" />
+{% endblock %}
+
+{% block content %}
+ {% for post in list.object_list %}
+ <article class="post">
+ (<span class="keyword">defpost</span>
+ <a href="{{ post.link }}"
+ class="variable-name">{{ post.title|slugify|nameless|truncate:48 }}</a>
+ (<img src="/static/images/logos/{{ post.icon }}" width="16"
+ height="16"/>)
+ <div class="doc">
+ {% autoescape off %}{{ post.content }}{% endautoescape %}
+ </div>
+ <span class="string">&quot;{{ post.updated }}&quot;</span>)
+ </article>
+ {% endfor %}
+
+<div id="pager">
+ {% if list.has_previous %}
+ <span class="nav-prev">
+ <a href="/{{ category }}/{{ list.previous_page_number }}/"
+ id="previous">[previous]</a>
+ </span>
+ {% endif %}
+
+ <span id="current">
+ {{ list.number }} / {{ list.paginator.num_pages }}
+ </span>
+
+ {% if list.has_next %}
+ <span class="nav-next">
+ <a href="/{{ category }}/{{ list.next_page_number }}/"
+ id="next">[next]</a>
+ </span>
+ {% endif %}
+</div>
+{% endblock %}
diff --git a/ryuslash/aggregator/templatetags/__init__.py b/ryuslash/aggregator/templatetags/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ryuslash/aggregator/templatetags/__init__.py
diff --git a/ryuslash/aggregator/templatetags/posts_extras.py b/ryuslash/aggregator/templatetags/posts_extras.py
new file mode 100644
index 0000000..64718a2
--- /dev/null
+++ b/ryuslash/aggregator/templatetags/posts_extras.py
@@ -0,0 +1,20 @@
+import re
+
+from django import template
+from django.template.defaultfilters import stringfilter
+
+register = template.Library()
+
+@stringfilter
+def nameless(value):
+ return re.sub(r'(^|by[- ])(ryuslash|tom)[- ]?', '', value)
+
+@stringfilter
+def truncate(value, length):
+ if len(value) > length:
+ value = value[:length-3] + '...'
+
+ return value
+
+register.filter('nameless', nameless)
+register.filter('truncate', truncate)
diff --git a/ryuslash/aggregator/views.py b/ryuslash/aggregator/views.py
new file mode 100644
index 0000000..bb3c1b7
--- /dev/null
+++ b/ryuslash/aggregator/views.py
@@ -0,0 +1,22 @@
+from django.core.paginator import Paginator, InvalidPage, EmptyPage
+from django.http import Http404
+from django.shortcuts import render
+
+from .models import Post
+
+def posts(request, cat, page=1):
+ category = cat or 'post'
+ queryset = Post.objects.filter(category=category)
+ paginator = Paginator(queryset, 20)
+
+ if page == None:
+ page = 1
+
+ try:
+ object_list = paginator.page(page)
+ except (EmptyPage, InvalidPage):
+ raise Http404
+
+ return render(request, 'aggregator/posts.html',
+ { 'list': object_list,
+ 'category': category })
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-06-02 12:26:21 +0000