refactored so the data model lives in its own module

models/__init__.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+#coding: utf-8
+
+#from sqlalchemy import create_engine, Table, Column, Integer, Text, String, Boolean, DateTime, MetaData, ForeignKey, desc
+#from sqlalchemy.orm import sessionmaker, relation, backref
+from sqlalchemy.ext.declarative import declarative_base
+#from datetime import datetime, timedelta
+#from time import mktime
+#import feedparser
+#import re
+#import sys
+#import urllib
+#import urllib2
+#import hn
+#import html2text
+#import ConfigParser
+#import pprint
+#from optparse import OptionParser
+#from cStringIO import StringIO
+#from email.mime.multipart import MIMEMultipart
+#from email.mime.text import MIMEText
+#from email.header import Header
+#from email import Charset
+#from email.generator import Generator
+#import smtplib
+
+Base = declarative_base()
+
+from models.feed import Feed
+from models.feedinfo import Feedinfo
+from models.entry import Entry

models/entry.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+#coding: utf-8
+
+#from sqlalchemy import create_engine, Table, Column, Integer, Text, String, Boolean, DateTime, MetaData, ForeignKey, desc
+from sqlalchemy import Column, Integer, ForeignKey, String, Text, DateTime
+#from sqlalchemy.orm import sessionmaker, relation, backref
+from sqlalchemy.orm import relation, backref
+#from sqlalchemy.ext.declarative import declarative_base
+#from datetime import datetime, timedelta
+from datetime import datetime
+from time import mktime
+#import feedparser
+#import re
+#import sys
+#import urllib
+#import urllib2
+#import hn
+#import html2text
+#import ConfigParser
+#import pprint
+#from optparse import OptionParser
+#from cStringIO import StringIO
+#from email.mime.multipart import MIMEMultipart
+#from email.mime.text import MIMEText
+#from email.header import Header
+#from email import Charset
+#from email.generator import Generator
+#import smtplib
+
+from models import Base
+
+class Entry(Base):
+    __tablename__ = 'entry'
+
+    id = Column(Integer, primary_key=True)
+    feed_id = Column(Integer, ForeignKey('feed.id'))
+    feed = relation("Feed", backref=backref('entry'))
+    title = Column(String(255))
+    link = Column(String(255))
+    summary = Column(Text)
+    content = Column(Text)
+    author = Column(String(255))
+    enclosures = Column(Text)
+
+    resolvedlink = Column(String(255))
+    fullpage = Column(Text)
+    readability = Column(Text)
+    updated = Column(DateTime)
+    firstfetched = Column(DateTime)
+    lastfetched = Column(DateTime)
+    sent = Column(DateTime)
+
+    def __init__(self, entry):
+        self.update(entry)
+        self.firstfetched = datetime.now()
+
+    def __repr__(self):
+       return "<Entry('%s','%s','%s')>" % (self.title, "", "")
+
+    def update(self, entry):
+        if entry.has_key('title'):
+            self.title = entry.get('title')
+        if entry.has_key('link'):
+            self.link = entry.get('link')
+        if entry.has_key('summary'):
+            self.summary = entry.get('summary')
+        if entry.has_key('content'):
+            self.content = entry.get('content')[0].value
+        if entry.has_key('author'):
+            self.author = entry.get('author')
+        if entry.has_key('updated_parsed'):
+            updated_parsed = entry.get('updated_parsed')
+            self.updated = datetime.fromtimestamp(mktime(updated_parsed))
+        if entry.has_key('enclosures') and len(entry.get('enclosures')) > 0:
+            print 'enclosures';
+            pp=pprint.PrettyPrinter(depth=4)
+            pp.pprint(entry.get('enclosures'))
+            #self.enclosures = entry.get('enclosures')
+        self.lastfetched = datetime.now()

models/feed.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+#coding: utf-8
+
+#from sqlalchemy import create_engine, Table, Column, Integer, Text, String, Boolean, DateTime, MetaData, ForeignKey, desc
+from sqlalchemy import Column, Integer, String, Boolean
+#from sqlalchemy.orm import sessionmaker, relation, backref
+#from sqlalchemy.ext.declarative import declarative_base
+#from datetime import datetime, timedelta
+#from time import mktime
+#import feedparser
+#import re
+#import sys
+#import urllib
+#import urllib2
+#import hn
+#import html2text
+#import ConfigParser
+#import pprint
+#from optparse import OptionParser
+#from cStringIO import StringIO
+#from email.mime.multipart import MIMEMultipart
+#from email.mime.text import MIMEText
+#from email.header import Header
+#from email import Charset
+#from email.generator import Generator
+#import smtplib
+
+from models import Base
+
+class Feed(Base):
+    __tablename__ = 'feed'
+
+    id = Column(Integer, primary_key=True)
+    url = Column(String(255))
+    frequency = Column(Integer)
+    daily = Column(Boolean)
+    resolveredirects = Column(Boolean)
+    readability = Column(Boolean)
+    fullpage = Column(Boolean)
+    html2textsummary = Column(Boolean)
+    html2textignoreimages = Column(Boolean)
+    enabled = Column(Boolean)
+
+    def __init__(self, url, daily, readability, fullpage, enabled, html2textsummary):
+        self.url = url
+        self.daily = daily
+        self.readability = readability
+        self.fullpage = fullpage
+        self.html2textsummary = html2textsummary
+        self.enabled = enabled
+
+    def __repr__(self):
+       return "<Feed('%s','%s','%s')>" % (self.url, self.daily, self.readability)

models/feedinfo.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+#coding: utf-8
+
+#from sqlalchemy import create_engine, Table, Column, Integer, Text, String, Boolean, DateTime, MetaData, ForeignKey, desc
+from sqlalchemy import Column, Integer, ForeignKey, String, DateTime
+#from sqlalchemy.orm import sessionmaker, relation, backref
+from sqlalchemy.orm import relation, backref
+#from sqlalchemy.ext.declarative import declarative_base
+#from datetime import datetime, timedelta
+from datetime import datetime
+#from time import mktime
+#import feedparser
+#import re
+#import sys
+#import urllib
+#import urllib2
+#import hn
+#import html2text
+#import ConfigParser
+#import pprint
+#from optparse import OptionParser
+#from cStringIO import StringIO
+#from email.mime.multipart import MIMEMultipart
+#from email.mime.text import MIMEText
+#from email.header import Header
+#from email import Charset
+#from email.generator import Generator
+#import smtplib
+
+from models import Base
+
+class Feedinfo(Base):
+    __tablename__ = 'feedinfo'
+
+    id = Column(Integer, primary_key=True)
+    feed_id = Column(Integer, ForeignKey('feed.id'))
+    feed = relation("Feed", backref=backref('feedinfo', uselist=False))
+    title = Column(String(255))
+    link = Column(String(255))
+    subtitle = Column(String(255))
+    author = Column(String(255))
+    publisher = Column(String(255))
+    status = Column(Integer)
+    version = Column(String(16))
+    encoding = Column(String(16))
+    bozo = Column(Integer)
+
+    lastfetched = Column(DateTime)
+    lastsuccessful = Column(DateTime)
+
+    def __init__(self, parser):
+        self.update(parser)
+
+    def __repr__(self):
+       return "<Feedinfo('%s','%s','%s')>" % (self.title, self.subtitle, self.author)
+
+    def update(self, parser):
+        if parser.feed.has_key('title'):
+            self.title = parser.feed.get('title')
+        if parser.feed.has_key('link'):
+            self.link = parser.feed.get('link')
+        if parser.feed.has_key('subtitle'):
+            self.subtitle = parser.feed.get('subtitle')
+        if parser.feed.has_key('author'):
+            self.author = parser.feed.get('author')
+        if parser.feed.has_key('publisher'):
+            self.author = parser.feed.get('publisher')
+        self.status = parser.get('status')
+        self.version = parser.get('version')
+        self.encoding = parser.get('encoding')
+        self.bozo = parser.get('bozo')
+        self.lastfetched = datetime.now()
+        if parser.get('status') == 200 or parser.get('status') == 302:
+            self.lastsuccessful = datetime.now()