pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: use UPSERT statements 

Avoid potential race conditions
master
pictuga 2020-05-03 21:27:45 +02:00
parent 73b477665e
commit f685139137
1 changed files with 5 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
morss/crawler.py
View File

@ -585,14 +585,8 @@ class SQLiteCache(BaseCache):
value[3] = sqlite3.Binary(value[3]) # data value[3] = sqlite3.Binary(value[3]) # data
value = tuple(value) value = tuple(value)
if url in self: with self.con:
with self.con: self.con.execute('INSERT INTO data VALUES (?,?,?,?,?,?) ON CONFLICT(url) DO UPDATE SET code=?, msg=?, headers=?, data=?, timestamp=?', (url,) + value + value)
self.con.execute('UPDATE data SET code=?, msg=?, headers=?, data=?, timestamp=? WHERE url=?',
value + (url,))
else:
with self.con:
self.con.execute('INSERT INTO data VALUES (?,?,?,?,?,?)', (url,) + value)
import pymysql.cursors import pymysql.cursors
@ -622,14 +616,9 @@ class MySQLCacheHandler(BaseCache):
return row[1:] return row[1:]
def __setitem__(self, url, value): # (code, msg, headers, data, timestamp) def __setitem__(self, url, value): # (code, msg, headers, data, timestamp)
if url in self: with self.cursor() as cursor:
with self.cursor() as cursor: cursor.execute('INSERT INTO data VALUES (%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE code=%s, msg=%s, headers=%s, data=%s, timestamp=%s',
cursor.execute('UPDATE data SET code=%s, msg=%s, headers=%s, data=%s, timestamp=%s WHERE url=%s', (url,) + value + value)
value + (url,))
else:
with self.cursor() as cursor:
cursor.execute('INSERT INTO data VALUES (%s,%s,%s,%s,%s,%s)', (url,) + value)
if __name__ == '__main__': if __name__ == '__main__':