pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: use BLOB in sqlite and drop "buffer" 

Can't really remember why "buffer" was introduced in the first place
master
pictuga 2017-11-04 13:54:40 +01:00
parent 203ba10dbd
commit b7db78f631
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
morss/crawler.py
View File

@ -22,7 +22,6 @@ try:
basestring
except NameError:
basestring = unicode = str
buffer = memoryview
MIMETYPE = {
@ -337,7 +336,7 @@ class CacheHandler(BaseHandler):
return out
def save(self, url, code, msg, headers, data, timestamp):
self.cache[url] = (code, msg, unicode(headers), buffer(data), timestamp)
self.cache[url] = (code, msg, unicode(headers), data, timestamp)
def http_request(self, req):
(code, msg, headers, data, timestamp) = self.load(req.get_full_url())
@ -481,7 +480,7 @@ class SQLiteCache:
self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False)
with self.con:
self.con.execute('CREATE TABLE IF NOT EXISTS data (url UNICODE PRIMARY KEY, code INT, msg UNICODE, headers UNICODE, data BYTES, timestamp INT)')
self.con.execute('CREATE TABLE IF NOT EXISTS data (url UNICODE PRIMARY KEY, code INT, msg UNICODE, headers UNICODE, data BLOB, timestamp INT)')
self.con.execute('pragma journal_mode=WAL')
def __del__(self):