From 385f9eb39a6d2721df7c9edc7fde98646d88f601 Mon Sep 17 00:00:00 2001 From: pictuga Date: Wed, 8 Mar 2017 19:05:48 -1000 Subject: [PATCH] morss: use crawler strict accept for feed --- morss/morss.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/morss/morss.py b/morss/morss.py index 3124f5e..fb3eb89 100644 --- a/morss/morss.py +++ b/morss/morss.py @@ -131,10 +131,10 @@ default_handlers = [crawler.GZIPHandler(), crawler.UAHandler(DEFAULT_UA), crawler.AutoRefererHandler(), crawler.HTTPEquivHandler(), crawler.HTTPRefreshHandler()] -def custom_handler(accept, delay=DELAY, encoding=None): +def custom_handler(accept, strict=False, delay=DELAY, encoding=None): handlers = default_handlers[:] handlers.append(crawler.EncodingFixHandler(encoding)) - handlers.append(crawler.ContentNegociationHandler(crawler.MIMETYPE[accept])) + handlers.append(crawler.ContentNegociationHandler(crawler.MIMETYPE[accept], strict)) handlers.append(crawler.SQliteCacheHandler(delay)) return build_opener(*handlers) @@ -267,7 +267,7 @@ def ItemFill(item, options, feedurl='/', fast=False): delay = -2 try: - con = custom_handler('html', delay, options.encoding).open(link, timeout=TIMEOUT) + con = custom_handler('html', False, delay, options.encoding).open(link, timeout=TIMEOUT) data = con.read() except (IOError, HTTPException) as e: @@ -368,7 +368,7 @@ def FeedFetch(url, options): delay = 0 try: - con = custom_handler('xml', delay, options.encoding).open(url, timeout=TIMEOUT * 2) + con = custom_handler('xml', True, delay, options.encoding).open(url, timeout=TIMEOUT * 2) xml = con.read() except (HTTPError) as e: