From 0df6409b0eaa5eb9f05db7bf2441fe68a92954fc Mon Sep 17 00:00:00 2001 From: pictuga Date: Sat, 28 Oct 2017 01:28:47 +0200 Subject: [PATCH] crawler: use `with con` to commit, journal WAL for perf --- morss/crawler.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/morss/crawler.py b/morss/crawler.py index bdae931..a85fbb8 100644 --- a/morss/crawler.py +++ b/morss/crawler.py @@ -488,8 +488,10 @@ class SQliteCacheHandler(BaseCacheHandler): BaseCacheHandler.__init__(self, force_min) self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False) - self.con.execute('create table if not exists data (url unicode PRIMARY KEY, code int, msg unicode, headers unicode, data bytes, timestamp int)') - self.con.commit() + + with self.con: + self.con.execute('create table if not exists data (url unicode PRIMARY KEY, code int, msg unicode, headers unicode, data bytes, timestamp int)') + self.con.execute('pragma journal_mode=WAL') def __del__(self): self.con.close() @@ -506,10 +508,12 @@ class SQliteCacheHandler(BaseCacheHandler): data = buffer(data) if self.con.execute('select code from data where url=?', (url,)).fetchone(): - self.con.execute('update data set code=?, msg=?, headers=?, data=?, timestamp=? where url=?', - (code, msg, headers, data, timestamp, url)) + with self.con: + self.con.execute('update data set code=?, msg=?, headers=?, data=?, timestamp=? where url=?', + (code, msg, headers, data, timestamp, url)) else: - self.con.execute('insert into data values (?,?,?,?,?,?)', (url, code, msg, headers, data, timestamp)) + with self.con: + self.con.execute('insert into data values (?,?,?,?,?,?)', (url, code, msg, headers, data, timestamp)) + - self.con.commit()