crawler: fix chinese encoding

2020-04-20 16:14:55 +02:00
parent 285e1e5f42
commit 2719bd6776
1 changed files with 9 additions and 0 deletions
--- a/morss/crawler.py
+++ b/morss/crawler.py
@@ -149,6 +149,15 @@ class GZIPHandler(BaseHandler):


 def detect_encoding(data, resp=None):
+    enc = detect_raw_encoding(data, resp)
+
+    if enc == 'gb2312':
+        enc = 'gbk'
+
+    return enc
+
+
+def detect_raw_encoding(data, resp=None):
    if resp is not None:
        enc = resp.headers.get('charset')
        if enc is not None: