pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Add "Feed/Item" in functions name 

To make it instantly clearer what they work on
master
pictuga 2017-02-28 23:23:15 -10:00
parent 60e3311e97
commit fcc5e8a076
1 changed files with 27 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
morss/morss.py
View File

@ -143,7 +143,7 @@ def custom_handler(accept, delay=DELAY):
return build_opener(*handlers) return build_opener(*handlers)
def Fix(item, feedurl='/'): def ItemFix(item, feedurl='/'):
""" Improves feed items (absolute links, resolve feedburner links, etc) """ """ Improves feed items (absolute links, resolve feedburner links, etc) """
# check unwanted uppercase title # check unwanted uppercase title
@ -213,7 +213,7 @@ def Fix(item, feedurl='/'):
return item return item
def Fill(item, options, feedurl='/', fast=False): def ItemFill(item, options, feedurl='/', fast=False):
""" Returns True when it has done its best """ """ Returns True when it has done its best """
if not item.link: if not item.link:
@ -294,7 +294,7 @@ def Fill(item, options, feedurl='/', fast=False):
return True return True
def Fetch(url, options): def FeedFetch(url, options):
# basic url clean-up # basic url clean-up
if url is None: if url is None:
raise MorssException('No url provided') raise MorssException('No url provided')
@ -336,7 +336,7 @@ def Fetch(url, options):
if url.startswith('https://itunes.apple.com/lookup?id='): if url.startswith('https://itunes.apple.com/lookup?id='):
link = json.loads(xml.decode('utf-8', 'replace'))['results'][0]['feedUrl'] link = json.loads(xml.decode('utf-8', 'replace'))['results'][0]['feedUrl']
log('itunes redirect: %s' % link) log('itunes redirect: %s' % link)
return Fetch(link, options) return FeedFetch(link, options)
elif re.match(b'\s*<?xml', xml) is not None or contenttype in MIMETYPE['xml']: elif re.match(b'\s*<?xml', xml) is not None or contenttype in MIMETYPE['xml']:
rss = feeds.parse(xml) rss = feeds.parse(xml)
@ -352,7 +352,7 @@ def Fetch(url, options):
if len(match): if len(match):
link = urljoin(url, match[0]) link = urljoin(url, match[0])
log('rss redirect: %s' % link) log('rss redirect: %s' % link)
return Fetch(link, options) return FeedFetch(link, options)
else: else:
log('no-link html') log('no-link html')
raise MorssException('Link provided is an HTML page, which doesn\'t link to a feed') raise MorssException('Link provided is an HTML page, which doesn\'t link to a feed')
@ -364,7 +364,7 @@ def Fetch(url, options):
return rss return rss
def Gather(rss, url, options): def FeedGather(rss, url, options):
size = len(rss.items) size = len(rss.items)
start_time = time.time() start_time = time.time()
@ -397,16 +397,16 @@ def Gather(rss, url, options):
item.remove() item.remove()
return return
item = Fix(item, url) item = ItemFix(item, url)
if time.time() - start_time > max_time >= 0 or i + 1 > max_item >= 0: if time.time() - start_time > max_time >= 0 or i + 1 > max_item >= 0:
if not options.proxy: if not options.proxy:
if Fill(item, options, url, True) is False: if ItemFill(item, options, url, True) is False:
item.remove() item.remove()
return return
else: else:
if not options.proxy: if not options.proxy:
Fill(item, options, url) ItemFill(item, options, url)
queue = Queue() queue = Queue()
@ -437,7 +437,7 @@ def Gather(rss, url, options):
return rss return rss
def Before(rss, options): def FeedBefore(rss, options):
for i, item in enumerate(list(rss.items)): for i, item in enumerate(list(rss.items)):
if options.empty: if options.empty:
item.remove() item.remove()
@ -451,7 +451,7 @@ def Before(rss, options):
return rss return rss
def After(rss, options): def FeedAfter(rss, options):
for i, item in enumerate(list(rss.items)): for i, item in enumerate(list(rss.items)):
if options.strip: if options.strip:
del item.desc del item.desc
@ -486,7 +486,7 @@ def After(rss, options):
return rss return rss
def Format(rss, options): def FeedFormat(rss, options):
if options.callback: if options.callback:
if re.match(r'^[a-zA-Z0-9\.]+$', options.callback) is not None: if re.match(r'^[a-zA-Z0-9\.]+$', options.callback) is not None:
return '%s(%s)' % (options.callback, rss.tojson()) return '%s(%s)' % (options.callback, rss.tojson())
@ -514,12 +514,12 @@ def process(url, cache=None, options=None):
options = Options(options) options = Options(options)
if cache: crawler.sqlite_default = cache if cache: crawler.sqlite_default = cache
rss = Fetch(url, options) rss = FeedFetch(url, options)
rss = Before(rss, options) rss = FeedBefore(rss, options)
rss = Gather(rss, url, options) rss = FeedGather(rss, url, options)
rss = After(rss, options) rss = FeedAfter(rss, options)
return Format(rss, options) return FeedFormat(rss, options)
def cgi_app(environ, start_response): def cgi_app(environ, start_response):
@ -572,17 +572,17 @@ def cgi_app(environ, start_response):
crawler.sqlite_default = os.path.join(os.getcwd(), 'morss-cache.db') crawler.sqlite_default = os.path.join(os.getcwd(), 'morss-cache.db')
# get the work done # get the work done
rss = Fetch(url, options) rss = FeedFetch(url, options)
if headers['content-type'] == 'text/xml': if headers['content-type'] == 'text/xml':
headers['content-type'] = rss.mimetype headers['content-type'] = rss.mimetype
start_response(headers['status'], list(headers.items())) start_response(headers['status'], list(headers.items()))
rss = Before(rss, options) rss = FeedBefore(rss, options)
rss = Gather(rss, url, options) rss = FeedGather(rss, url, options)
rss = After(rss, options) rss = FeedAfter(rss, options)
out = Format(rss, options) out = FeedFormat(rss, options)
if not options.silent: if not options.silent:
return out return out
@ -647,11 +647,11 @@ def cli_app():
crawler.sqlite_default = os.path.expanduser('~/.cache/morss-cache.db') crawler.sqlite_default = os.path.expanduser('~/.cache/morss-cache.db')
rss = Fetch(url, options) rss = FeedFetch(url, options)
rss = Before(rss, options) rss = FeedBefore(rss, options)
rss = Gather(rss, url, options) rss = FeedGather(rss, url, options)
rss = After(rss, options) rss = FeedAfter(rss, options)
out = Format(rss, options) out = FeedFormat(rss, options)
if not options.silent: if not options.silent:
print(out.decode('utf-8', 'replace') if isinstance(out, bytes) else out) print(out.decode('utf-8', 'replace') if isinstance(out, bytes) else out)