pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: 301 http code doesn't respect headers 

More or less according to the specs
master
pictuga 2017-03-18 22:18:10 -10:00
parent f4abc4e8a4
commit 9ee6ff60e1
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
morss/crawler.py
View File

@ -294,6 +294,12 @@ class BaseCacheHandler(BaseHandler):
# force refresh # force refresh
return None return None
elif code == 301 and cache_age < 7*24*3600:
print('301 so cached')
# "301 Moved Permanently" has to be cached...as long as we want (awesome HTTP specs), let's say a week (why not?)
# use force_min=0 if you want to bypass this (needed for a proper refresh)
pass
elif self.force_min is None and ('no-cache' in cc_list elif self.force_min is None and ('no-cache' in cc_list
or 'no-store' in cc_list or 'no-store' in cc_list
or ('private' in cc_list and not self.private)): or ('private' in cc_list and not self.private)):
@ -308,11 +314,6 @@ class BaseCacheHandler(BaseHandler):
# still recent enough for us, use cache # still recent enough for us, use cache
pass pass
elif code == 301 and cache_age < 7*24*3600:
# "301 Moved Permanently" has to be cached...as long as we want (awesome HTTP specs), let's say a week (why not?)
# use force_min=0 if you want to bypass this (needed for a proper refresh)
pass
else: else:
# according to the www, we have to refresh when nothing is said # according to the www, we have to refresh when nothing is said
return None return None