Compare commits

..

2 Commits

Author SHA1 Message Date
pictuga 961a31141f morss: fix url fixing 2020-03-21 17:28:00 +01:00
pictuga a7b01ee85e readabilite: further html processing instructions fix 2020-03-21 17:23:50 +01:00
2 changed files with 12 additions and 6 deletions

View File

@ -284,24 +284,26 @@ def ItemAfter(item, options):
return item return item
def FeedFetch(url, options): def UrlFix(url):
# basic url clean-up
if url is None: if url is None:
raise MorssException('No url provided') raise MorssException('No url provided')
if isinstance(url, bytes):
url = url.decode()
if urlparse(url).scheme not in PROTOCOL: if urlparse(url).scheme not in PROTOCOL:
url = 'http://' + url url = 'http://' + url
log(url) log(url)
url = url.replace(' ', '%20') url = url.replace(' ', '%20')
if isinstance(url, bytes): return url
url = url.decode()
def FeedFetch(url, options):
# allow for code execution for feedify # allow for code execution for feedify
pre = feedify.pre_worker(url) pre = feedify.pre_worker(url)
if pre: if pre:
url = pre url = UrlFix(pre)
log('url redirect') log('url redirect')
log(url) log(url)
@ -324,7 +326,7 @@ def FeedFetch(url, options):
if options.items: if options.items:
# using custom rules # using custom rules
rss = feeds.FeedHTML(xml, url, contenttype) rss = feeds.FeedHTML(xml)
feed.rule feed.rule
rss.rules['items'] = options.items rss.rules['items'] = options.items
@ -475,6 +477,7 @@ def process(url, cache=None, options=None):
if cache: if cache:
crawler.default_cache = crawler.SQLiteCache(cache) crawler.default_cache = crawler.SQLiteCache(cache)
url = UrlFix(url)
rss = FeedFetch(url, options) rss = FeedFetch(url, options)
rss = FeedGather(rss, url, options) rss = FeedGather(rss, url, options)
@ -537,6 +540,7 @@ def cgi_app(environ, start_response):
crawler.default_cache = crawler.SQLiteCache(os.path.join(os.getcwd(), 'morss-cache.db')) crawler.default_cache = crawler.SQLiteCache(os.path.join(os.getcwd(), 'morss-cache.db'))
# get the work done # get the work done
url = UrlFix(url)
rss = FeedFetch(url, options) rss = FeedFetch(url, options)
if headers['content-type'] == 'text/xml': if headers['content-type'] == 'text/xml':
@ -608,6 +612,7 @@ def cli_app():
crawler.default_cache = crawler.SQLiteCache(os.path.expanduser('~/.cache/morss-cache.db')) crawler.default_cache = crawler.SQLiteCache(os.path.expanduser('~/.cache/morss-cache.db'))
url = UrlFix(url)
rss = FeedFetch(url, options) rss = FeedFetch(url, options)
rss = FeedGather(rss, url, options) rss = FeedGather(rss, url, options)
out = FeedFormat(rss, options) out = FeedFormat(rss, options)

View File

@ -93,6 +93,7 @@ def score_node(node):
class_id = node.get('class', '') + node.get('id', '') class_id = node.get('class', '') + node.get('id', '')
if (isinstance(node, lxml.html.HtmlComment) if (isinstance(node, lxml.html.HtmlComment)
or isinstance(node, lxml.html.HtmlProcessingInstruction)
or node.tag in tags_bad or node.tag in tags_bad
or regex_bad.search(class_id)): or regex_bad.search(class_id)):
return 0 return 0