pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Move from gzip to zlib to decompress data 

Faster on incomplete files
master
pictuga 2017-11-25 19:57:41 +01:00
parent d091e74d56
commit 21480f90de
1 changed files with 4 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
morss/crawler.py
View File

@ -3,7 +3,7 @@ import sys
import ssl import ssl
import socket import socket
from gzip import GzipFile import zlib
from io import BytesIO, StringIO from io import BytesIO, StringIO
import re import re
import chardet import chardet
@ -100,22 +100,9 @@ class SizeLimitHandler(BaseHandler):
https_response = http_response https_response = http_response
def UnGzip(cprss, CHUNKSIZE=64*1024): # the bigger the CHUNKSIZE, the faster def UnGzip(data):
" Supports truncated files " " Supports truncated files "
gz = GzipFile(fileobj=cprss, mode='rb') return zlib.decompressobj(zlib.MAX_WBITS | 32).decompress(data)
data = b''
chunk = gz.read(CHUNKSIZE)
try:
while chunk:
data += chunk
chunk = gz.read(CHUNKSIZE)
except (IOError, EOFError):
pass
return data
class GZIPHandler(BaseHandler): class GZIPHandler(BaseHandler):
@ -128,7 +115,7 @@ class GZIPHandler(BaseHandler):
if resp.headers.get('Content-Encoding') == 'gzip': if resp.headers.get('Content-Encoding') == 'gzip':
data = resp.read() data = resp.read()
data = UnGzip(BytesIO(data)) data = UnGzip(data)
resp.headers['Content-Encoding'] = 'identity' resp.headers['Content-Encoding'] = 'identity'