From c952b85d9236db23f0fd8b3069c2e15cbec5a9c2 Mon Sep 17 00:00:00 2001 From: pictuga Date: Thu, 9 Mar 2017 09:37:05 -1000 Subject: [PATCH] crawler: cache 301 HTTP code, for a week --- morss/crawler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/morss/crawler.py b/morss/crawler.py index 6898412..5827a69 100644 --- a/morss/crawler.py +++ b/morss/crawler.py @@ -308,6 +308,11 @@ class BaseCacheHandler(BaseHandler): # still recent enough for us, use cache pass + elif code == 301 and cache_age < 7*24*3600: + # "301 Moved Permanently" has to be cached...as long as we want (awesome HTTP specs), let's say a week (why not?) + # use force_min=0 if you want to bypass this (needed for a proper refresh) + pass + else: # according to the www, we have to refresh when nothing is said return None