diff --git a/atomstrom.py b/atomstrom.py index 8eff05e..79220ff 100755 --- a/atomstrom.py +++ b/atomstrom.py @@ -6,6 +6,7 @@ from sqlalchemy.ext.declarative import declarative_base import datetime import feedparser import re +import sys Base = declarative_base() @@ -16,12 +17,14 @@ class Feed(Base): url = Column(Text) daily = Column(Boolean) readability = Column(Boolean) + fullpage = Column(Boolean) enabled = Column(Boolean) - def __init__(self, url, daily, readability, enabled): + def __init__(self, url, daily, readability, fullpage, enabled): self.url = url self.daily = daily self.readability = readability + self.fullpage = fullpage self.enabled = enabled def __repr__(self): @@ -43,10 +46,10 @@ class Feedinfo(Base): version = Column(Text) encoding = Column(Text) bozo = Column(Integer) + lastfetched = Column(DateTime) lastsuccessful = Column(DateTime) - def __init__(self, parser): self.update(parser) @@ -85,7 +88,10 @@ class Entry(Base): content = Column(Text) author = Column(Text) enclosures = Column(Text) + + fullpage = Column(Text) lastfetched = Column(DateTime) + sent = Column(DateTime) def __init__(self, entry): self.update(entry) @@ -115,12 +121,13 @@ Base.metadata.create_all(engine) Session = sessionmaker(bind=engine) session = Session() -#session.add(Feed('http://www.heise.de/newsticker/heise-atom.xml', 1, 0, 0)) -#session.add(Feed('http://blog.schatenseite.de/feed/', 1, 0, 1)) +#session.add(Feed('http://www.heise.de/newsticker/heise-atom.xml', 1, 0, 0, 1)) +#session.add(Feed('http://blog.schatenseite.de/feed/', 1, 0, 0, 1)) for feed in session.query(Feed).filter_by(enabled=1).order_by(Feed.id): print "fetching %s" % feed.url parser = feedparser.parse(feed.url) + print "processing feed info..." query = session.query(Feedinfo).filter(Feedinfo.feed_id==feed.id) try: feed.feedinfo = query.one() @@ -128,12 +135,15 @@ for feed in session.query(Feed).filter_by(enabled=1).order_by(Feed.id): except Exception, e: feed.feedinfo = Feedinfo(parser) + print "processing feed entries: ", for entry in parser.entries: query = session.query(Entry).filter_by(feed_id=feed.id, title=entry.title.encode('latin-1', 'replace')) try: thisentry = query.one() + sys.stdout.write("-") thisentry.update(entry) except Exception, e: + sys.stdout.write("+") feed.entry.append(Entry(entry)) print