morss/feeds.py

@@ -21,10 +21,12 @@ json.encoder.c_make_encoder = None
 try:
     # python 2
     from StringIO import StringIO
+    from urllib2 import urlopen
     from ConfigParser import RawConfigParser
 except ImportError:
     # python 3
     from io import StringIO
+    from urllib.request import urlopen
     from configparser import RawConfigParser
 
 try:
@@ -162,7 +164,7 @@ class ParserBase(object):
         return self.convert(FeedHTML).tostring(**k)
 
     def convert(self, TargetParser):
-        if type(self) == TargetParser:
+        if isinstance(self, TargetParser):
             return self
 
         target = TargetParser()
@@ -206,11 +208,11 @@ class ParserBase(object):
         pass
 
     def rule_remove(self, rule):
-        # remove node from its parent. Returns nothing
+        # remove node from its parent
         pass
 
     def rule_set(self, rule, value):
-        # set the value. Returns nothing
+        # value is always a str?
         pass
 
     def rule_str(self, rule):
@@ -245,30 +247,25 @@ class ParserBase(object):
 
         return self.rule_search_all(self.rules[rule_name])
 
-    def get(self, rule_name):
+    def get_str(self, rule_name):
         # simple function to get nice text from the rule name
-        # for use in @property, ie. self.get('title')
+        # for use in @property, ie. self.get_str('title')
         if rule_name not in self.rules:
             return None
 
-        return self.rule_str(self.rules[rule_name]) or None
+        return self.rule_str(self.rules[rule_name])
 
-    def set(self, rule_name, value):
-        # simple function to set nice text from the rule name. Returns nothing
+    def set_str(self, rule_name, value):
         if rule_name not in self.rules:
-            return
-
-        if value is None:
-            self.rmv(rule_name)
-            return
+            return None
 
         try:
-            self.rule_set(self.rules[rule_name], value)
+            return self.rule_set(self.rules[rule_name], value)
 
         except AttributeError:
             # does not exist, have to create it
             self.rule_create(self.rules[rule_name])
-            self.rule_set(self.rules[rule_name], value)
+            return self.rule_set(self.rules[rule_name], value)
 
     def rmv(self, rule_name):
         # easy deleter
@@ -372,6 +369,10 @@ class ParserXML(ParserBase):
             match.getparent().append(element)
             return element
 
+        # try duplicating from template
+        # FIXME
+        # >>> self.xml.getroottree().getpath(ff.find('a'))
+
         return None
 
     def rule_remove(self, rule):
@@ -431,7 +432,7 @@ class ParserXML(ParserBase):
                 return etree.tostring(match, method='text', encoding='unicode').strip()
 
         else:
-            return match # might be None is no match
+            return match or ""
 
 
 class ParserHTML(ParserXML):
@@ -467,6 +468,8 @@ class ParserHTML(ParserXML):
             element = deepcopy(match)
             match.getparent().append(element)
 
+    # TODO def rule_set for the html part
+
 
 def parse_time(value):
     if isinstance(value, basestring):
@@ -481,7 +484,6 @@ def parse_time(value):
 
     elif isinstance(value, datetime):
         return value
-
     else:
         return False
 
@@ -495,9 +497,8 @@ class ParserJSON(ParserBase):
         return json.loads(raw)
 
     def remove(self):
-        # impossible to "delete" oneself per se but can clear all its items
-        for attr in self.root:
-            del self.root[attr]
+        # delete oneself FIXME
+        pass
 
     def tostring(self, encoding='unicode', **k):
         dump = json.dumps(self.root, ensure_ascii=False, **k) # ensure_ascii = False to have proper (unicode) string and not \u00
@@ -557,15 +558,10 @@ class ParserJSON(ParserBase):
         rrule = self._rule_parse(rule)
         cur = self.root
 
-        try:
-            for node in rrule[:-1]:
-                cur = cur[node]
+        for node in rrule[:-1]:
+            cur = cur[node]
 
-            del cur[rrule[-1]]
-
-        except KeyError:
-            # nothing to delete
-            pass
+        del cur[rrule[-1]]
 
     def rule_set(self, rule, value):
         if '[]' in rule:
@@ -613,12 +609,12 @@ class Feed(object):
         return [itemsClass(x, self.rules, self) for x in items]
 
     title = property(
-        lambda f:   f.get('title'),
-        lambda f,x: f.set('title', x),
+        lambda f:   f.get_str('title'),
+        lambda f,x: f.set_str('title', x),
         lambda f:   f.rmv('title') )
     description = desc = property(
-        lambda f:   f.get('desc'),
-        lambda f,x: f.set('desc', x),
+        lambda f:   f.get_str('desc'),
+        lambda f,x: f.set_str('desc', x),
         lambda f:   f.rmv('desc') )
     items = property(
         lambda f:   f )
@@ -665,28 +661,28 @@ class Item(Uniq):
         return id(xml)
 
     title = property(
-        lambda f:   f.get('item_title'),
-        lambda f,x: f.set('item_title', x),
+        lambda f:   f.get_str('item_title'),
+        lambda f,x: f.set_str('item_title', x),
         lambda f:   f.rmv('item_title') )
     link = property(
-        lambda f:   f.get('item_link'),
-        lambda f,x: f.set('item_link', x),
+        lambda f:   f.get_str('item_link'),
+        lambda f,x: f.set_str('item_link', x),
         lambda f:   f.rmv('item_link') )
     description = desc = property(
-        lambda f:   f.get('item_desc'),
-        lambda f,x: f.set('item_desc', x),
+        lambda f:   f.get_str('item_desc'),
+        lambda f,x: f.set_str('item_desc', x),
         lambda f:   f.rmv('item_desc') )
     content = property(
-        lambda f:   f.get('item_content'),
-        lambda f,x: f.set('item_content', x),
+        lambda f:   f.get_str('item_content'),
+        lambda f,x: f.set_str('item_content', x),
         lambda f:   f.rmv('item_content') )
     time = property(
-        lambda f:   f.time_prs(f.get('item_time')),
-        lambda f,x: f.set('item_time', f.time_fmt(x)),
+        lambda f:   f.time_prs(f.get_str('item_time')),
+        lambda f,x: f.set_str('item_time', f.time_fmt(x)),
         lambda f:   f.rmv('item_time') )
     updated = property(
-        lambda f:   f.time_prs(f.get('item_updated')),
-        lambda f,x: f.set('item_updated', f.time_fmt(x)),
+        lambda f:   f.time_prs(f.get_str('item_updated')),
+        lambda f,x: f.set_str('item_updated', f.time_fmt(x)),
         lambda f:   f.rmv('item_updated') )
 
 

morss/morss.py

@@ -204,7 +204,7 @@ def ItemFill(item, options, feedurl='/', fast=False):
 
     # twitter
     if urlparse(feedurl).netloc == 'twitter.com':
-        match = lxml.html.fromstring(item.desc).xpath('//a/@data-expanded-url')
+        match = lxml.html.fromstring(item.content).xpath('//a/@data-expanded-url')
         if len(match):
             link = match[0]
             log(link)
@@ -341,8 +341,6 @@ def FeedFetch(url, options):
     else:
         try:
             rss = feeds.parse(xml, url, contenttype)
-            rss = rss.convert(feeds.FeedXML)
-                # contains all fields, otherwise much-needed data can be lost
 
         except TypeError:
             log('random page')
@@ -437,10 +435,8 @@ def FeedFormat(rss, options):
     if options.callback:
         if re.match(r'^[a-zA-Z0-9\.]+$', options.callback) is not None:
             return '%s(%s)' % (options.callback, rss.tojson())
-
         else:
             raise MorssException('Invalid callback var name')
-
     elif options.json:
         if options.indent:
             return rss.tojson(encoding='UTF-8', indent=4)