rschaten/Atomstrom
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

completed data model, added some debug-output

Browse Source
This commit is contained in:
Ronald Schaten 2010-10-29 00:13:00 +02:00
parent aaf0d493ed
commit f979f32bae
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
atomstrom.py
View File

@ -6,6 +6,7 @@ from sqlalchemy.ext.declarative import declarative_base
import datetime import datetime
import feedparser import feedparser
import re import re
import sys
Base = declarative_base() Base = declarative_base()
@ -16,12 +17,14 @@ class Feed(Base):
url = Column(Text) url = Column(Text)
daily = Column(Boolean) daily = Column(Boolean)
readability = Column(Boolean) readability = Column(Boolean)
fullpage = Column(Boolean)
enabled = Column(Boolean) enabled = Column(Boolean)
def __init__(self, url, daily, readability, enabled): def __init__(self, url, daily, readability, fullpage, enabled):
self.url = url self.url = url
self.daily = daily self.daily = daily
self.readability = readability self.readability = readability
self.fullpage = fullpage
self.enabled = enabled self.enabled = enabled
def __repr__(self): def __repr__(self):
@ -43,10 +46,10 @@ class Feedinfo(Base):
version = Column(Text) version = Column(Text)
encoding = Column(Text) encoding = Column(Text)
bozo = Column(Integer) bozo = Column(Integer)
lastfetched = Column(DateTime) lastfetched = Column(DateTime)
lastsuccessful = Column(DateTime) lastsuccessful = Column(DateTime)
def __init__(self, parser): def __init__(self, parser):
self.update(parser) self.update(parser)
@ -85,7 +88,10 @@ class Entry(Base):
content = Column(Text) content = Column(Text)
author = Column(Text) author = Column(Text)
enclosures = Column(Text) enclosures = Column(Text)
fullpage = Column(Text)
lastfetched = Column(DateTime) lastfetched = Column(DateTime)
sent = Column(DateTime)
def __init__(self, entry): def __init__(self, entry):
self.update(entry) self.update(entry)
@ -115,12 +121,13 @@ Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine) Session = sessionmaker(bind=engine)
session = Session() session = Session()
#session.add(Feed('http://www.heise.de/newsticker/heise-atom.xml', 1, 0, 0)) #session.add(Feed('http://www.heise.de/newsticker/heise-atom.xml', 1, 0, 0, 1))
#session.add(Feed('http://blog.schatenseite.de/feed/', 1, 0, 1)) #session.add(Feed('http://blog.schatenseite.de/feed/', 1, 0, 0, 1))
for feed in session.query(Feed).filter_by(enabled=1).order_by(Feed.id): for feed in session.query(Feed).filter_by(enabled=1).order_by(Feed.id):
print "fetching %s" % feed.url print "fetching %s" % feed.url
parser = feedparser.parse(feed.url) parser = feedparser.parse(feed.url)
print "processing feed info..."
query = session.query(Feedinfo).filter(Feedinfo.feed_id==feed.id) query = session.query(Feedinfo).filter(Feedinfo.feed_id==feed.id)
try: try:
feed.feedinfo = query.one() feed.feedinfo = query.one()
@ -128,12 +135,15 @@ for feed in session.query(Feed).filter_by(enabled=1).order_by(Feed.id):
except Exception, e: except Exception, e:
feed.feedinfo = Feedinfo(parser) feed.feedinfo = Feedinfo(parser)
print "processing feed entries: ",
for entry in parser.entries: for entry in parser.entries:
query = session.query(Entry).filter_by(feed_id=feed.id, title=entry.title.encode('latin-1', 'replace')) query = session.query(Entry).filter_by(feed_id=feed.id, title=entry.title.encode('latin-1', 'replace'))
try: try:
thisentry = query.one() thisentry = query.one()
sys.stdout.write("-")
thisentry.update(entry) thisentry.update(entry)
except Exception, e: except Exception, e:
sys.stdout.write("+")
feed.entry.append(Entry(entry)) feed.entry.append(Entry(entry))
print print