109 lines
3.8 KiB
Python
109 lines
3.8 KiB
Python
from sqlalchemy import Column, Integer, String, Boolean, Enum
|
|
from sqlalchemy.orm import relationship, backref
|
|
from datetime import datetime
|
|
import feedparser
|
|
import sys
|
|
|
|
from models import Base
|
|
from models.feedinfo import Feedinfo
|
|
from models.entry import Entry
|
|
|
|
|
|
class Feed(Base):
|
|
__tablename__ = 'feed'
|
|
|
|
id = Column(Integer, primary_key=True)
|
|
url = Column(String(255))
|
|
frequency = Column(Integer)
|
|
daily = Column(Boolean)
|
|
resolveredirects = Column(Boolean)
|
|
readability = Column(Boolean)
|
|
fullpage = Column(Boolean)
|
|
contentcolumn = Column(Enum('summary', 'content', 'fullpage', 'readability'))
|
|
html2textcontent = Column(Boolean)
|
|
html2textignorelinks = Column(Boolean)
|
|
html2textignoreimages = Column(Boolean)
|
|
enabled = Column(Boolean)
|
|
entries = relationship("Entry", backref=backref('feed'), lazy='dynamic', cascade='all, delete, delete-orphan')
|
|
feedinfo = relationship("Feedinfo", backref=backref('feed'), cascade='all, delete, delete-orphan', uselist=False)
|
|
|
|
def __init__(self, url, daily, readability, fullpage, enabled, html2textcontent):
|
|
self.url = url
|
|
self.daily = daily
|
|
self.readability = readability
|
|
self.fullpage = fullpage
|
|
self.html2textcontent = html2textcontent
|
|
self.enabled = enabled
|
|
|
|
def __unicode__(self):
|
|
id = self.id
|
|
if self.feedinfo:
|
|
title = self.feedinfo.title
|
|
last = self.feedinfo.lastsuccessful
|
|
else:
|
|
title = '<unknown>'
|
|
last = '<never>'
|
|
if self.enabled:
|
|
enabled = 'enabled'
|
|
else:
|
|
enabled = 'DISABLED'
|
|
entries = len(self.entries)
|
|
url = self.url
|
|
return u'%3d %s (%d entries, last fetched %s, %s)\n %s' % (id, title, entries, last, enabled, url)
|
|
|
|
def __str__(self):
|
|
return unicode(self).encode('utf-8')
|
|
|
|
def __repr__(self):
|
|
return "<Feed('%d','%s')>" % (self.id, self.url)
|
|
|
|
def fetch(self):
|
|
print 'processing %d: %s' % (self.id, self.url)
|
|
fetched = False
|
|
if self.feedinfo:
|
|
if (not self.feedinfo.nextfetch) or (self.feedinfo.nextfetch < datetime.now()):
|
|
print 'feed known, fetching...'
|
|
try:
|
|
parser = feedparser.parse(self.url)
|
|
fetched = True
|
|
self.feedinfo.update(parser)
|
|
except:
|
|
print 'ERROR parsing feed'
|
|
print sys.exc_info()
|
|
else:
|
|
print 'not fetching before: %s' % self.feedinfo.nextfetch
|
|
else:
|
|
print 'feed seems to be new, fetching...'
|
|
try:
|
|
parser = feedparser.parse(self.url)
|
|
fetched = True
|
|
self.feedinfo = Feedinfo(parser)
|
|
except:
|
|
print 'ERROR parsing feed'
|
|
print sys.exc_info()
|
|
|
|
if fetched:
|
|
print 'processing feed entries:'
|
|
entries_new = 0
|
|
entries_total = 0
|
|
for entry in parser.entries:
|
|
entries_total += 1
|
|
thisentry = self.entries.\
|
|
filter(Entry.title == entry.title).\
|
|
filter(Entry.link == entry.link).\
|
|
first()
|
|
if thisentry:
|
|
print ' entry already known <%s>' % entry.title
|
|
thisentry.lastfetched = datetime.now()
|
|
else:
|
|
print ' new entry <%s>' % entry.title
|
|
self.entries.append(Entry(entry, self))
|
|
entries_new += 1
|
|
print 'updated %d of %d entries' % (entries_new, entries_total)
|
|
|
|
def reset(self):
|
|
self.entries[:] = []
|
|
self.feedinfo = None
|
|
|
|
# -*- coding: utf-8 -*-
|