pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: use BLOB in sqlite and drop "buffer" 

Can't really remember why "buffer" was introduced in the first place
master
pictuga 2017-11-04 13:54:40 +01:00
parent 203ba10dbd
commit b7db78f631
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
morss/crawler.py
View File

@ -22,7 +22,6 @@ try:
basestring basestring
except NameError: except NameError:
basestring = unicode = str basestring = unicode = str
buffer = memoryview
MIMETYPE = { MIMETYPE = {
@ -337,7 +336,7 @@ class CacheHandler(BaseHandler):
return out return out
def save(self, url, code, msg, headers, data, timestamp): def save(self, url, code, msg, headers, data, timestamp):
self.cache[url] = (code, msg, unicode(headers), buffer(data), timestamp) self.cache[url] = (code, msg, unicode(headers), data, timestamp)
def http_request(self, req): def http_request(self, req):
(code, msg, headers, data, timestamp) = self.load(req.get_full_url()) (code, msg, headers, data, timestamp) = self.load(req.get_full_url())
@ -481,7 +480,7 @@ class SQLiteCache:
self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False) self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False)
with self.con: with self.con:
self.con.execute('CREATE TABLE IF NOT EXISTS data (url UNICODE PRIMARY KEY, code INT, msg UNICODE, headers UNICODE, data BYTES, timestamp INT)') self.con.execute('CREATE TABLE IF NOT EXISTS data (url UNICODE PRIMARY KEY, code INT, msg UNICODE, headers UNICODE, data BLOB, timestamp INT)')
self.con.execute('pragma journal_mode=WAL') self.con.execute('pragma journal_mode=WAL')
def __del__(self): def __del__(self):