pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: clean up encoding detection code

master
pictuga 2020-05-27 21:35:24 +02:00
parent dcfdb75a15
commit ce4cf01aa6
1 changed files with 3 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
morss/crawler.py
View File

@ -261,14 +261,10 @@ class EncodingFixHandler(BaseHandler):
if 200 <= resp.code < 300 and maintype == 'text': if 200 <= resp.code < 300 and maintype == 'text':
data = resp.read() data = resp.read()
if not self.encoding: enc = self.encoding or detect_encoding(data, resp)
enc = detect_encoding(data, resp)
else:
enc = self.encoding
if enc: data = data.decode(enc, 'replace')
data = data.decode(enc, 'replace') data = data.encode(enc)
data = data.encode(enc)
fp = BytesIO(data) fp = BytesIO(data)
old_resp = resp old_resp = resp