pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

morss: fix url fixing

master
pictuga 2020-03-21 17:28:00 +01:00
parent a7b01ee85e
commit 961a31141f
1 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
morss/morss.py
View File

@ -284,24 +284,26 @@ def ItemAfter(item, options):
return item return item
def FeedFetch(url, options): def UrlFix(url):
# basic url clean-up
if url is None: if url is None:
raise MorssException('No url provided') raise MorssException('No url provided')
if isinstance(url, bytes):
url = url.decode()
if urlparse(url).scheme not in PROTOCOL: if urlparse(url).scheme not in PROTOCOL:
url = 'http://' + url url = 'http://' + url
log(url) log(url)
url = url.replace(' ', '%20') url = url.replace(' ', '%20')
if isinstance(url, bytes): return url
url = url.decode()
def FeedFetch(url, options):
# allow for code execution for feedify # allow for code execution for feedify
pre = feedify.pre_worker(url) pre = feedify.pre_worker(url)
if pre: if pre:
url = pre url = UrlFix(pre)
log('url redirect') log('url redirect')
log(url) log(url)
@ -324,7 +326,7 @@ def FeedFetch(url, options):
if options.items: if options.items:
# using custom rules # using custom rules
rss = feeds.FeedHTML(xml, url, contenttype) rss = feeds.FeedHTML(xml)
feed.rule feed.rule
rss.rules['items'] = options.items rss.rules['items'] = options.items
@ -475,6 +477,7 @@ def process(url, cache=None, options=None):
if cache: if cache:
crawler.default_cache = crawler.SQLiteCache(cache) crawler.default_cache = crawler.SQLiteCache(cache)
url = UrlFix(url)
rss = FeedFetch(url, options) rss = FeedFetch(url, options)
rss = FeedGather(rss, url, options) rss = FeedGather(rss, url, options)
@ -537,6 +540,7 @@ def cgi_app(environ, start_response):
crawler.default_cache = crawler.SQLiteCache(os.path.join(os.getcwd(), 'morss-cache.db')) crawler.default_cache = crawler.SQLiteCache(os.path.join(os.getcwd(), 'morss-cache.db'))
# get the work done # get the work done
url = UrlFix(url)
rss = FeedFetch(url, options) rss = FeedFetch(url, options)
if headers['content-type'] == 'text/xml': if headers['content-type'] == 'text/xml':
@ -608,6 +612,7 @@ def cli_app():
crawler.default_cache = crawler.SQLiteCache(os.path.expanduser('~/.cache/morss-cache.db')) crawler.default_cache = crawler.SQLiteCache(os.path.expanduser('~/.cache/morss-cache.db'))
url = UrlFix(url)
rss = FeedFetch(url, options) rss = FeedFetch(url, options)
rss = FeedGather(rss, url, options) rss = FeedGather(rss, url, options)
out = FeedFormat(rss, options) out = FeedFormat(rss, options)