pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: fix chinese encoding

master
pictuga 2020-04-20 16:14:55 +02:00
parent 285e1e5f42
commit 2719bd6776
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
morss/crawler.py
View File

@ -149,6 +149,15 @@ class GZIPHandler(BaseHandler):
def detect_encoding(data, resp=None): def detect_encoding(data, resp=None):
enc = detect_raw_encoding(data, resp)
if enc == 'gb2312':
enc = 'gbk'
return enc
def detect_raw_encoding(data, resp=None):
if resp is not None: if resp is not None:
enc = resp.headers.get('charset') enc = resp.headers.get('charset')
if enc is not None: if enc is not None: