pictuga/morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: use with con to commit, journal WAL for perf

Browse Source
This commit is contained in:
pictuga
2017-10-28 01:28:47 +02:00
parent 7b85f692a0
commit 0df6409b0e
1 changed files with 10 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
morss/crawler.py
View File

@@ -488,8 +488,10 @@ class SQliteCacheHandler(BaseCacheHandler):
BaseCacheHandler.__init__(self, force_min) BaseCacheHandler.__init__(self, force_min)
self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False) self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False)
self.con.execute('create table if not exists data (url unicode PRIMARY KEY, code int, msg unicode, headers unicode, data bytes, timestamp int)')
self.con.commit() with self.con:
self.con.execute('create table if not exists data (url unicode PRIMARY KEY, code int, msg unicode, headers unicode, data bytes, timestamp int)')
self.con.execute('pragma journal_mode=WAL')
def __del__(self): def __del__(self):
self.con.close() self.con.close()
@@ -506,10 +508,12 @@ class SQliteCacheHandler(BaseCacheHandler):
data = buffer(data) data = buffer(data)
if self.con.execute('select code from data where url=?', (url,)).fetchone(): if self.con.execute('select code from data where url=?', (url,)).fetchone():
self.con.execute('update data set code=?, msg=?, headers=?, data=?, timestamp=? where url=?', with self.con:
(code, msg, headers, data, timestamp, url)) self.con.execute('update data set code=?, msg=?, headers=?, data=?, timestamp=? where url=?',
(code, msg, headers, data, timestamp, url))
else: else:
self.con.execute('insert into data values (?,?,?,?,?,?)', (url, code, msg, headers, data, timestamp)) with self.con:
self.con.execute('insert into data values (?,?,?,?,?,?)', (url, code, msg, headers, data, timestamp))
self.con.commit()