pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Make Fill return True when it had done sth useful

master
pictuga 2013-05-15 17:38:52 +02:00
parent 9694a31052
commit bf647ba5f8
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
morss.py
View File

@ -317,12 +317,14 @@ def EncDownload(url):
return (data, enc, con.geturl()) return (data, enc, con.geturl())
def Fill(rss, cache, feedurl="/", mode='feed'): def Fill(rss, cache, feedurl="/", mode='feed'):
""" Returns True when it has done its best """
item = XMLMap(rss, ITEM_MAP, True) item = XMLMap(rss, ITEM_MAP, True)
log(item.link) log(item.link)
if 'link' not in item: if 'link' not in item:
log('no link') log('no link')
return return True
# feedburner and others # feedburner and others
if '{http://rssnamespace.org/feedburner/ext/1.0}origLink' in item: if '{http://rssnamespace.org/feedburner/ext/1.0}origLink' in item:
@ -347,7 +349,7 @@ def Fill(rss, cache, feedurl="/", mode='feed'):
log('content: %s vs %s' % (len_content, len_desc)) log('content: %s vs %s' % (len_content, len_desc))
if len_content > 5*len_desc: if len_content > 5*len_desc:
log('provided') log('provided')
return return True
# check cache and previous errors # check cache and previous errors
if item.link in cache: if item.link in cache:
@ -356,17 +358,17 @@ def Fill(rss, cache, feedurl="/", mode='feed'):
if match: if match:
if cache.isYoungerThan(DELAY*60): if cache.isYoungerThan(DELAY*60):
log('cached error: %s' % match.groups()[0]) log('cached error: %s' % match.groups()[0])
return return True
else: else:
log('old error') log('old error')
else: else:
log('cached') log('cached')
item.content = cache.get(item.link) item.content = cache.get(item.link)
return return True
# super-fast mode # super-fast mode
if mode == 'cache': if mode == 'cache':
return return False
# download # download
ddl = EncDownload(item.link) ddl = EncDownload(item.link)
@ -374,7 +376,7 @@ def Fill(rss, cache, feedurl="/", mode='feed'):
if ddl is False: if ddl is False:
log('http error') log('http error')
cache.set(item.link, 'error-http') cache.set(item.link, 'error-http')
return return True
data, enc, url = ddl data, enc, url = ddl
log(enc) log(enc)
@ -386,8 +388,9 @@ def Fill(rss, cache, feedurl="/", mode='feed'):
else: else:
log('not bigger enough') log('not bigger enough')
cache.set(item.link, 'error-length') cache.set(item.link, 'error-length')
return return True
return True
def Gather(url, cachePath, mode='feed'): def Gather(url, cachePath, mode='feed'):
cache = Cache(cachePath, url) cache = Cache(cachePath, url)