implemented fetch-frequency for feeds

atomstrom.py

@@ -3,7 +3,7 @@
 from sqlalchemy import create_engine, Table, Column, Integer, Text, Boolean, DateTime, MetaData, ForeignKey, desc
 from sqlalchemy.orm import sessionmaker, relation, backref
 from sqlalchemy.ext.declarative import declarative_base
-from datetime import datetime
+from datetime import datetime, timedelta
 from time import mktime
 import feedparser
 import re
@@ -12,6 +12,7 @@ import urllib
 import hn
 import html2text
 import ConfigParser
+import pprint
 import smtplib
 from email.mime.text import MIMEText
 from optparse import OptionParser
@@ -23,6 +24,7 @@ class Feed(Base):
 
     id = Column(Integer, primary_key=True)
     url = Column(Text)
+    frequency = Column(Integer)
     daily = Column(Boolean)
     readability = Column(Boolean)
     fullpage = Column(Boolean)
@@ -225,17 +227,27 @@ def process_feed_entry(session, feed, entry):
         return 1
 
 def fetch_single_feed(session, feed):
-    print 'fetching %s' % feed.url
-    parser = feedparser.parse(feed.url)
-    print 'processing feed info...'
+    print 'processing %s' % feed.url
     query = session.query(Feedinfo).filter(Feedinfo.feed_id==feed.id)
+    fetched = False
     try:
         feed.feedinfo = query.one()
+        nextfetch = (feed.feedinfo.lastfetched + timedelta(minutes=feed.frequency))
+        if datetime.now() > nextfetch:
+            print 'fetching...'
+            parser = feedparser.parse(feed.url)
+            fetched = True
             feed.feedinfo.update(parser)
+        else:
+            print 'not fetching before: %s' % nextfetch
     except Exception, e:
         print 'this feed seems to be new'
+        print 'fetching...'
+        parser = feedparser.parse(feed.url)
+        fetched = True
         feed.feedinfo = Feedinfo(parser)
 
+    if fetched:
         print 'processing feed entries:'
         entries_new = 0
         entries_total = 0