From 5ee5dbf359842ac3b529c86d960459a288baeea8 Mon Sep 17 00:00:00 2001 From: pictuga Date: Wed, 1 May 2013 17:56:03 +0200 Subject: [PATCH] Cache http errors to save time. --- morss.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/morss.py b/morss.py index 79103af..d907dc3 100644 --- a/morss.py +++ b/morss.py @@ -199,7 +199,10 @@ class XMLMap(object): return unicode(out) if self._str else out def __getitem__(self, tag): - return self.__getattr__(tag) + if self.__contains__(tag): + return self.__getattr__(tag) + else: + return None def __setattr__(self, tag, value): if tag.startswith('_'): @@ -251,7 +254,6 @@ def EncDownload(url): data = con.read() except (urllib2.HTTPError, urllib2.URLError) as error: log(error) - log('http error') return False # meta-redirect @@ -303,17 +305,27 @@ def Fill(rss, cache): item.link = "".join([(t[s[0]] if s[0] in t else "=") + s[1:] for s in url[1:]]) log(item.link) - # check cache + # check cache and previous errors if item.link in cache: - log('cached') - item.content = cache.get(item.link) - return + content = cache.get(item.link) + if content == 'httperr': + if cache.isYoungerThan(DELAY*60): + log('cached http err') + return + else: + log('old http error') + else: + log('cached') + item.content = cache.get(item.link) + return # download ddl = EncDownload(item.link) if ddl is False: - return item + log('http error') + cache.set(item.link, 'httperr') + return data, enc, url = ddl log(enc)