major refactoring: feed knows how to fetch itself

models/feed.py

@@ -3,8 +3,13 @@
 
 from sqlalchemy import Column, Integer, String, Boolean, Enum
 from sqlalchemy.orm import relationship, backref
+from datetime import datetime
+import feedparser
+import sys
 
 from models import Base
+from models.feedinfo import Feedinfo
+from models.entry import Entry
 
 
 class Feed(Base):
@@ -22,7 +27,7 @@ class Feed(Base):
     html2textignorelinks = Column(Boolean)
     html2textignoreimages = Column(Boolean)
     enabled = Column(Boolean)
-    entries = relationship("Entry", backref=backref('feed'), cascade='all, delete, delete-orphan')
+    entries = relationship("Entry", backref=backref('feed'), lazy='dynamic', cascade='all, delete, delete-orphan')
     feedinfo = relationship("Feedinfo", backref=backref('feed'), cascade='all, delete, delete-orphan', uselist=False)
 
     def __init__(self, url, daily, readability, fullpage, enabled, html2textcontent):
@@ -55,6 +60,50 @@ class Feed(Base):
     def __repr__(self):
         return "<Feed('%d','%s')>" % (self.id, self.url)
 
+    def fetch(self):
+        print 'processing %d: %s' % (self.id, self.url)
+        fetched = False
+        if self.feedinfo:
+            if (not self.feedinfo.nextfetch) or (self.feedinfo.nextfetch < datetime.now()):
+                print 'feed known, fetching...'
+                try:
+                    parser = feedparser.parse(self.url)
+                    fetched = True
+                    self.feedinfo.update(parser)
+                except:
+                    print 'ERROR parsing feed'
+                    print sys.exc_info()
+            else:
+                print 'not fetching before: %s' % self.feedinfo.nextfetch
+        else:
+            print 'feed seems to be new, fetching...'
+            try:
+                parser = feedparser.parse(self.url)
+                fetched = True
+                self.feedinfo = Feedinfo(parser)
+            except:
+                print 'ERROR parsing feed'
+                print sys.exc_info()
+
+        if fetched:
+            print 'processing feed entries:'
+            entries_new = 0
+            entries_total = 0
+            for entry in parser.entries:
+                entries_total += 1
+                thisentry = self.entries.\
+                            filter(Entry.title == entry.title).\
+                            filter(Entry.link == entry.link).\
+                            first()
+                if thisentry:
+                    print '  entry already known <%s>' % entry.title
+                    thisentry.lastfetched = datetime.now()
+                else:
+                    print '  new entry <%s>' % entry.title
+                    self.entries.append(Entry(entry, self))
+                    entries_new += 1
+            print 'updated %d of %d entries' % (entries_new, entries_total)
+
     def reset(self):
         self.entries[:] = []
         self.feedinfo = None