pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: make SQLiteCache inherit from BaseCache 

Saves some time for other cache backends
master
pictuga 2017-11-04 14:48:00 +01:00
parent 2d5bf7b38b
commit f29a107a09
1 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
morss/crawler.py
View File

@ -472,10 +472,22 @@ class CacheHandler(BaseHandler):
https_response = http_response https_response = http_response
class BaseCache:
def __contains__(self, url):
try:
self[url]
except KeyError:
return False
else:
return True
import sqlite3 import sqlite3
class SQLiteCache: class SQLiteCache(BaseCache):
def __init__(self, filename=':memory:'): def __init__(self, filename=':memory:'):
self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False) self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False)
@ -495,7 +507,7 @@ class SQLiteCache:
return row[1:] return row[1:]
def __setitem__(self, url, value): # value = (code, msg, headers, data, timestamp) def __setitem__(self, url, value): # value = (code, msg, headers, data, timestamp)
if self.con.execute('SELECT code FROM data WHERE url=?', (url,)).fetchone(): if url in self:
with self.con: with self.con:
self.con.execute('UPDATE data SET code=?, msg=?, headers=?, data=?, timestamp=? WHERE url=?', self.con.execute('UPDATE data SET code=?, msg=?, headers=?, data=?, timestamp=? WHERE url=?',
value + (url,)) value + (url,))