From df2239683873f5353abf886972d68d16f33e88bc Mon Sep 17 00:00:00 2001 From: pictuga Date: Sun, 16 Jul 2017 23:59:06 +0200 Subject: [PATCH] Only use chardet on 2k letters Takes forever otherwise --- morss/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morss/crawler.py b/morss/crawler.py index b1f3d88..2cafd4e 100644 --- a/morss/crawler.py +++ b/morss/crawler.py @@ -98,7 +98,7 @@ def detect_encoding(data, con=None): if match: return match.groups()[0].lower().decode() - enc = chardet.detect(data)['encoding'] + enc = chardet.detect(data[-2000:])['encoding'] if enc and enc != 'ascii': return enc