437 lines
12 KiB

# This file is part of morss
# Copyright (C) 2013-2020 pictuga <>
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <>.
import os
import re
import sys
import time
from datetime import datetime
from fnmatch import fnmatch
import lxml.etree
import lxml.html
from dateutil import tz
from . import caching, crawler, feeds, readabilite
# python 2
from httplib import HTTPException
from urlparse import parse_qs, urljoin, urlparse
except ImportError:
# python 3
from http.client import HTTPException
from urllib.parse import parse_qs, urljoin, urlparse
MAX_ITEM = int(os.getenv('MAX_ITEM', 5)) # cache-only beyond
MAX_TIME = int(os.getenv('MAX_TIME', 2)) # cache-only after (in sec)
LIM_ITEM = int(os.getenv('LIM_ITEM', 10)) # deletes what's beyond
LIM_TIME = int(os.getenv('LIM_TIME', 2.5)) # deletes what's after
DELAY = int(os.getenv('DELAY', 10 * 60)) # xml cache & ETag cache (in sec)
TIMEOUT = int(os.getenv('TIMEOUT', 4)) # http timeout (in sec)
class MorssException(Exception):
def log(txt):
if 'DEBUG' in os.environ:
if 'REQUEST_URI' in os.environ:
# when running on Apache
open('morss.log', 'a').write("%s\n" % repr(txt))
# when using internal server or cli
print(repr(txt), file=sys.stderr)
def len_html(txt):
if len(txt):
return len(lxml.html.fromstring(txt).text_content())
return 0
def count_words(txt):
if len(txt):
return len(lxml.html.fromstring(txt).text_content().split())
return 0
class Options:
def __init__(self, options=None, **args):
if len(args):
self.options = args
self.options.update(options or {})
self.options = options or {}
def __getattr__(self, key, default=None):
if key in self.options:
return self.options[key]
return default
def __setitem__(self, key, value):
self.options[key] = value
def __contains__(self, key):
return key in self.options
get = __getitem__ = __getattr__
def ItemFix(item, options, feedurl='/'):
""" Improves feed items (absolute links, resolve feedburner links, etc) """
# check unwanted uppercase title
if item.title is not None and len(item.title) > 20 and item.title.isupper():
item.title = item.title.title()
# check if it includes link
if not
log('no link')
return item
# wikipedia daily highlight
if fnmatch(feedurl, 'http*://**&feedformat=atom'):
match = lxml.html.fromstring(item.desc).xpath('//b/a/@href')
if len(match): = match[0]
# at user's election, use first <a>
if options.firstlink and (item.desc or item.content):
match = lxml.html.fromstring(item.desc or item.content).xpath('//a/@href')
if len(match): = match[0]
# check relative urls = urljoin(feedurl,
# google translate
if fnmatch(, '*/translate*u=*'): = parse_qs(urlparse(['u'][0]
# google
if fnmatch(, '*/url?q=*'): = parse_qs(urlparse(['q'][0]
# google news
if fnmatch(, '*url=*'): = parse_qs(urlparse(['url'][0]
# pocket
if fnmatch(, '*'): = parse_qs(urlparse(['url'][0]
# facebook
if fnmatch(, '*'): = parse_qs(urlparse(['u'][0]
# feedburner FIXME only works if RSS...
item.NSMAP['feedburner'] = ''
match = item.rule_str('feedburner:origLink')
if match: = match
# feedsportal
match ='/([0-9a-zA-Z]{20,})/story01.htm$',
if match:
url = match.groups()[0].split('0')
t = {'A': '0', 'B': '.', 'C': '/', 'D': '?', 'E': '-', 'F': '=',
'G': '&', 'H': ',', 'I': '_', 'J': '%', 'K': '+', 'L': 'http://',
'M': 'https://', 'N': '.com', 'O': '', 'P': ';', 'Q': '|',
'R': ':', 'S': 'www.', 'T': '#', 'U': '$', 'V': '~', 'W': '!',
'X': '(', 'Y': ')', 'Z': 'Z'} = ''.join([(t[s[0]] if s[0] in t else s[0]) + s[1:] for s in url[1:]])
# reddit
if urlparse(feedurl).netloc == '':
match = lxml.html.fromstring(item.content).xpath('//a[text()="[link]"]/@href')
if len(match): = match[0]
return item
def ItemFill(item, options, feedurl='/', fast=False):
""" Returns True when it has done its best """
if not
log('no link')
return True
# download
if fast or options.cache:
# force cache, don't fetch
policy = 'offline'
elif options.force:
# force refresh
policy = 'refresh'
policy = None
req = crawler.adv_get(, policy=policy, force_min=24*60*60, timeout=TIMEOUT)
except (IOError, HTTPException) as e:
log('http error')
return False # let's just delete errors stuff when in cache mode
if req['contenttype'] not in crawler.MIMETYPE['html'] and req['contenttype'] != 'text/plain':
log('non-text page')
return True
if not req['data']:
log('empty page')
return True
out = readabilite.get_article(req['data'], url=req['url'], encoding_in=req['encoding'], encoding_out='unicode', xpath=options.xpath)
if out is not None:
item.content = out
if options.resolve: = req['url']
return True
def ItemBefore(item, options):
# return None if item deleted
if not in item.title:
return None
return item
def ItemAfter(item, options):
if options.clip and item.desc and item.content:
item.content = item.desc + "<br/><br/><hr/><br/><br/>" + item.content
del item.desc
if options.nolink and item.content:
content = lxml.html.fromstring(item.content)
for link in content.xpath('//a'):
item.content = lxml.etree.tostring(content, method='html')
if options.noref: = ''
return item
def FeedFetch(url, options):
# fetch feed
delay = DELAY
if options.cache:
policy = 'offline'
elif options.force:
policy = 'refresh'
policy = None
req = crawler.adv_get(url=url,, follow=('rss' if not options.items else None), policy=policy, force_min=5*60, force_max=60*60, timeout=TIMEOUT)
except (IOError, HTTPException):
raise MorssException('Error downloading feed')
if options.items:
# using custom rules
ruleset = {}
ruleset['items'] = options.items
if options.mode:
ruleset['mode'] = options.mode
ruleset['title'] = options.get('title', '//head/title')
ruleset['desc'] = options.get('desc', '//head/meta[@name="description"]/@content')
ruleset['item_title'] = options.get('item_title', '.')
ruleset['item_link'] = options.get('item_link', '(.|.//a|ancestor::a)/@href')
if options.item_content:
ruleset['item_content'] = options.item_content
if options.item_time:
ruleset['item_time'] = options.item_time
rss = feeds.parse(req['data'], encoding=req['encoding'], ruleset=ruleset)
rss = rss.convert(feeds.FeedXML)
rss = feeds.parse(req['data'], url=url, encoding=req['encoding'])
rss = rss.convert(feeds.FeedXML)
# contains all fields, otherwise much-needed data can be lost
except TypeError:
log('random page')
raise MorssException('Link provided is not a valid feed')
return req['url'], rss
def FeedGather(rss, url, options):
size = len(rss.items)
start_time = time.time()
# custom settings
lim_item = LIM_ITEM
lim_time = LIM_TIME
max_item = MAX_ITEM
max_time = MAX_TIME
if options.cache:
max_time = 0
# sort
sorted_items = list(rss.items)
if options.order == 'last':
# `first` does nothing from a practical standpoint, so only `last` needs
# to be addressed
sorted_items = reversed(sorted_items)
elif options.order in ['newest', 'oldest']:
now =
sorted_items = sorted(sorted_items, key=lambda x:x.updated or x.time or now) # oldest to newest
if options.order == 'newest':
sorted_items = reversed(sorted_items)
for i, item in enumerate(sorted_items):
# hard cap
if time.time() - start_time > lim_time >= 0 or i + 1 > lim_item >= 0:
item = ItemBefore(item, options)
if item is None:
item = ItemFix(item, options, url)
# soft cap
if time.time() - start_time > max_time >= 0 or i + 1 > max_item >= 0:
if not options.proxy:
if ItemFill(item, options, url, True) is False:
if not options.proxy:
ItemFill(item, options, url)
item = ItemAfter(item, options)
new = rss.items.append()
new.title = "Are you hungry?"
new.desc = "Eat some Galler chocolate :)" = ""
new.time = "5 Oct 2013 22:42"
log(time.time() - start_time)
return rss
def FeedFormat(rss, options, encoding='utf-8'):
if options.callback:
if re.match(r'^[a-zA-Z0-9\.]+$', options.callback) is not None:
out = '%s(%s)' % (options.callback, rss.tojson(encoding='unicode'))
return out if encoding == 'unicode' else out.encode(encoding)
raise MorssException('Invalid callback var name')
elif options.format == 'json':
if options.indent:
return rss.tojson(encoding=encoding, indent=4)
return rss.tojson(encoding=encoding)
elif options.format == 'csv':
return rss.tocsv(encoding=encoding)
elif options.format == 'html':
if options.indent:
return rss.tohtml(encoding=encoding, pretty_print=True)
return rss.tohtml(encoding=encoding)
else: # i.e. format == 'rss'
if options.indent:
return rss.torss(xml_declaration=(not encoding == 'unicode'), encoding=encoding, pretty_print=True)
return rss.torss(xml_declaration=(not encoding == 'unicode'), encoding=encoding)
def process(url, cache=None, options=None):
if not options:
options = []
options = Options(options)
if cache:
caching.default_cache = caching.DiskCacheHandler(cache)
url, rss = FeedFetch(url, options)
rss = FeedGather(rss, url, options)
return FeedFormat(rss, options, 'unicode')