pictuga/morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: clean up encoding detection code

Browse Source
This commit is contained in:
pictuga
2020-05-27 21:35:24 +02:00
parent dcfdb75a15
commit ce4cf01aa6
1 changed files with 3 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
morss/crawler.py
View File

@@ -261,12 +261,8 @@ class EncodingFixHandler(BaseHandler):
if 200 <= resp.code < 300 and maintype == 'text': if 200 <= resp.code < 300 and maintype == 'text':
data = resp.read() data = resp.read()
if not self.encoding: enc = self.encoding or detect_encoding(data, resp)
enc = detect_encoding(data, resp)
else:
enc = self.encoding
if enc:
data = data.decode(enc, 'replace') data = data.decode(enc, 'replace')
data = data.encode(enc) data = data.encode(enc)