rschaten/Atomstrom
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

fuzzied time between feed fetches

Browse Source
This commit is contained in:
Ronald Schaten 2013-03-28 23:35:20 +01:00
parent 7789cb2051
commit d1d127744c
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
atomstrom.py
View File

@ -5,7 +5,7 @@ from models import Base, Feed, Feedinfo, Entry
from sqlalchemy import create_engine, desc from sqlalchemy import create_engine, desc
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
from datetime import datetime, timedelta from datetime import datetime
import feedparser import feedparser
import sys import sys
import codecs import codecs
@ -184,8 +184,7 @@ def fetch_single_feed(session, feed):
fetched = False fetched = False
if thisfeedinfo: if thisfeedinfo:
feed.feedinfo = thisfeedinfo feed.feedinfo = thisfeedinfo
nextfetch = (feed.feedinfo.lastfetched + timedelta(minutes=feed.frequency)) if (not feed.feedinfo.nextfetch) or (feed.feedinfo.nextfetch < datetime.now()):
if datetime.now() > nextfetch:
print 'feed known, fetching...' print 'feed known, fetching...'
try: try:
parser = feedparser.parse(feed.url) parser = feedparser.parse(feed.url)
@ -195,7 +194,7 @@ def fetch_single_feed(session, feed):
print 'ERROR parsing feed' print 'ERROR parsing feed'
print sys.exc_info() print sys.exc_info()
else: else:
print 'not fetching before: %s' % nextfetch print 'not fetching before: %s' % feed.feedinfo.nextfetch
else: else:
print 'feed seems to be new, fetching...' print 'feed seems to be new, fetching...'
try: try:

7
models/feedinfo.py
View File

@ -3,7 +3,8 @@
from sqlalchemy import Column, Integer, ForeignKey, String, DateTime from sqlalchemy import Column, Integer, ForeignKey, String, DateTime
from sqlalchemy.orm import relation, backref from sqlalchemy.orm import relation, backref
from datetime import datetime from datetime import datetime, timedelta
from random import randint
from models import Base from models import Base
@ -25,6 +26,7 @@ class Feedinfo(Base):
lastfetched = Column(DateTime) lastfetched = Column(DateTime)
lastsuccessful = Column(DateTime) lastsuccessful = Column(DateTime)
nextfetch = Column(DateTime)
def __init__(self, parser): def __init__(self, parser):
self.update(parser) self.update(parser)
@ -50,3 +52,6 @@ class Feedinfo(Base):
self.lastfetched = datetime.now() self.lastfetched = datetime.now()
if parser.get('status') == 200 or parser.get('status') == 302: if parser.get('status') == 200 or parser.get('status') == 302:
self.lastsuccessful = datetime.now() self.lastsuccessful = datetime.now()
waitminutes = randint(int(self.feed.frequency * .75), int(self.feed.frequency * 1.25))
self.nextfetch = (datetime.now() + timedelta(minutes=waitminutes))