pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: use `with con` to commit, journal WAL for perf

master
pictuga 2017-10-28 01:28:47 +02:00
parent 7b85f692a0
commit 0df6409b0e
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
morss/crawler.py
View File

@ -488,8 +488,10 @@ class SQliteCacheHandler(BaseCacheHandler):
BaseCacheHandler.__init__(self, force_min) BaseCacheHandler.__init__(self, force_min)
self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False) self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False)
with self.con:
self.con.execute('create table if not exists data (url unicode PRIMARY KEY, code int, msg unicode, headers unicode, data bytes, timestamp int)') self.con.execute('create table if not exists data (url unicode PRIMARY KEY, code int, msg unicode, headers unicode, data bytes, timestamp int)')
self.con.commit() self.con.execute('pragma journal_mode=WAL')
def __del__(self): def __del__(self):
self.con.close() self.con.close()
@ -506,10 +508,12 @@ class SQliteCacheHandler(BaseCacheHandler):
data = buffer(data) data = buffer(data)
if self.con.execute('select code from data where url=?', (url,)).fetchone(): if self.con.execute('select code from data where url=?', (url,)).fetchone():
with self.con:
self.con.execute('update data set code=?, msg=?, headers=?, data=?, timestamp=? where url=?', self.con.execute('update data set code=?, msg=?, headers=?, data=?, timestamp=? where url=?',
(code, msg, headers, data, timestamp, url)) (code, msg, headers, data, timestamp, url))
else: else:
with self.con:
self.con.execute('insert into data values (?,?,?,?,?,?)', (url, code, msg, headers, data, timestamp)) self.con.execute('insert into data values (?,?,?,?,?,?)', (url, code, msg, headers, data, timestamp))
self.con.commit()