crawler: clean up encoding detection code

master
pictuga 2020-05-27 21:35:24 +02:00
parent dcfdb75a15
commit ce4cf01aa6
1 changed files with 3 additions and 7 deletions

View File

@ -261,14 +261,10 @@ class EncodingFixHandler(BaseHandler):
if 200 <= resp.code < 300 and maintype == 'text':
data = resp.read()
if not self.encoding:
enc = detect_encoding(data, resp)
else:
enc = self.encoding
enc = self.encoding or detect_encoding(data, resp)
if enc:
data = data.decode(enc, 'replace')
data = data.encode(enc)
data = data.decode(enc, 'replace')
data = data.encode(enc)
fp = BytesIO(data)
old_resp = resp