Only use chardet on 2k letters

Takes forever otherwise
2017-07-16 23:59:06 +02:00
parent 6f0efd5802
commit df22396838
1 changed files with 1 additions and 1 deletions
--- a/morss/crawler.py
+++ b/morss/crawler.py
@@ -98,7 +98,7 @@ def detect_encoding(data, con=None):
    if match:
        return match.groups()[0].lower().decode()

-    enc = chardet.detect(data)['encoding']
+    enc = chardet.detect(data[-2000:])['encoding']
    if enc and enc != 'ascii':
        return enc