pictuga/morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: drop encoding setting

Browse Source
This commit is contained in:
pictuga
2020-10-30 22:41:16 +01:00
parent ad927e03a7
commit 3e886caaab
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
morss/crawler.py
View File

@@ -104,7 +104,7 @@ def adv_get(url, timeout=None, *args, **kwargs):
} }
def custom_handler(follow=None, delay=None, encoding=None): def custom_handler(follow=None, delay=None):
handlers = [] handlers = []
# as per urllib2 source code, these Handelers are added first # as per urllib2 source code, these Handelers are added first
@@ -123,7 +123,7 @@ def custom_handler(follow=None, delay=None, encoding=None):
handlers.append(HTTPRefreshHandler()) handlers.append(HTTPRefreshHandler())
handlers.append(UAHandler(random.choice(DEFAULT_UAS))) handlers.append(UAHandler(random.choice(DEFAULT_UAS)))
handlers.append(BrowserlyHeaderHandler()) handlers.append(BrowserlyHeaderHandler())
handlers.append(EncodingFixHandler(encoding)) handlers.append(EncodingFixHandler())
if follow: if follow:
handlers.append(AlternateHandler(MIMETYPE[follow])) handlers.append(AlternateHandler(MIMETYPE[follow]))