fuzzied time between feed fetches

atomstrom.py

@@ -5,7 +5,7 @@ from models import Base, Feed, Feedinfo, Entry
 
 from sqlalchemy import create_engine, desc
 from sqlalchemy.orm import sessionmaker
-from datetime import datetime, timedelta
+from datetime import datetime
 import feedparser
 import sys
 import codecs
@@ -184,8 +184,7 @@ def fetch_single_feed(session, feed):
     fetched = False
     if thisfeedinfo:
         feed.feedinfo = thisfeedinfo
-        nextfetch = (feed.feedinfo.lastfetched + timedelta(minutes=feed.frequency))
-        if datetime.now() > nextfetch:
+        if (not feed.feedinfo.nextfetch) or (feed.feedinfo.nextfetch < datetime.now()):
             print 'feed known, fetching...'
             try:
                 parser = feedparser.parse(feed.url)
@@ -195,7 +194,7 @@ def fetch_single_feed(session, feed):
                 print 'ERROR parsing feed'
                 print sys.exc_info()
         else:
-            print 'not fetching before: %s' % nextfetch
+            print 'not fetching before: %s' % feed.feedinfo.nextfetch
     else:
         print 'feed seems to be new, fetching...'
         try:

models/feedinfo.py

@@ -3,7 +3,8 @@
 
 from sqlalchemy import Column, Integer, ForeignKey, String, DateTime
 from sqlalchemy.orm import relation, backref
-from datetime import datetime
+from datetime import datetime, timedelta
+from random import randint
 
 from models import Base
 
@@ -25,6 +26,7 @@ class Feedinfo(Base):
 
     lastfetched = Column(DateTime)
     lastsuccessful = Column(DateTime)
+    nextfetch = Column(DateTime)
 
     def __init__(self, parser):
         self.update(parser)
@@ -50,3 +52,6 @@ class Feedinfo(Base):
         self.lastfetched = datetime.now()
         if parser.get('status') == 200 or parser.get('status') == 302:
             self.lastsuccessful = datetime.now()
+        waitminutes = randint(int(self.feed.frequency * .75), int(self.feed.frequency * 1.25))
+        self.nextfetch = (datetime.now() + timedelta(minutes=waitminutes))
+