pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Change :cache mode behavior 

Makes underlying code way cleaner
master
pictuga 2015-04-07 09:38:22 +08:00
parent cafb87d561
commit 131ba09207
2 changed files with 8 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
morss/crawler.py
View File

@ -203,7 +203,7 @@ class BaseCacheHandler(BaseHandler):
handler_order = 499 handler_order = 499
def __init__(self, force_min=None): def __init__(self, force_min=None):
self.force_min = force_min # force_min (seconds) to bypass http headers, -1 forever, 0 never, -2 do nothing if not in cache, -3 is like -2 but raises an error self.force_min = force_min # force_min (seconds) to bypass http headers, -1 forever, 0 never, -2 do nothing if not in cache
def _load(self, url): def _load(self, url):
out = list(self.load(url)) out = list(self.load(url))
@ -254,21 +254,16 @@ class BaseCacheHandler(BaseHandler):
cache_age = time.time() - timestamp cache_age = time.time() - timestamp
# list in a simple way what to do when # list in a simple way what to do when
if self.force_min in (-2, -3): if self.force_min == -2:
if code is not None: if code is not None:
# already in cache, perfect, use cache # already in cache, perfect, use cache
pass pass
else: else:
# ok then... headers['morss'] = 'from_cache'
if self.force_min == -2: resp = addinfourl(BytesIO(), headers, req.get_full_url(), 409)
headers['morss'] = 'from_cache' resp.msg = 'Conflict'
resp = addinfourl(BytesIO(), headers, req.get_full_url(), 409) return resp
resp.msg = 'Conflict'
return resp
elif self.force_min == -3:
raise NotInCache()
elif code is None: elif code is None:
# cache empty, refresh # cache empty, refresh

8
morss/morss.py
View File

@ -272,19 +272,15 @@ def Fill(item, options, feedurl='/', fast=False):
if fast: if fast:
# super-fast mode # super-fast mode
delay = -3 delay = -2
try: try:
con = custom_handler(('html', 'text/*'), delay).open(link, timeout=TIMEOUT) con = custom_handler(('html', 'text/*'), delay).open(link, timeout=TIMEOUT)
data = con.read() data = con.read()
except crawler.NotInCache:
log('skipped')
return False
except (IOError, HTTPException) as e: except (IOError, HTTPException) as e:
log('http error') log('http error')
return True return False # let's just delete errors stuff when in cache mode
contenttype = con.info().get('Content-Type', '').split(';')[0] contenttype = con.info().get('Content-Type', '').split(';')[0]
if contenttype not in MIMETYPE['html'] and contenttype != 'text/plain': if contenttype not in MIMETYPE['html'] and contenttype != 'text/plain':