pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: drop 'basic' toggle 

Can't even remember the use case
master
pictuga 2020-04-05 16:03:06 +02:00
parent e6811138fd
commit 8a4d68d72c
2 changed files with 3 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
morss/crawler.py
View File

@ -33,7 +33,7 @@ MIMETYPE = {
DEFAULT_UA = 'Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0'
def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=False):
def custom_handler(accept=None, strict=False, delay=None, encoding=None):
handlers = []
# as per urllib2 source code, these Handelers are added first
@ -51,9 +51,7 @@ def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=F
handlers.append(HTTPEquivHandler())
handlers.append(HTTPRefreshHandler())
handlers.append(UAHandler(DEFAULT_UA))
if not basic:
handlers.append(AutoRefererHandler())
handlers.append(AutoRefererHandler())
handlers.append(EncodingFixHandler(encoding))

2
morss/morss.py
View File

@ -336,7 +336,7 @@ def FeedFetch(url, options):
try:
con = crawler.custom_handler(accept='xml', strict=True, delay=delay,
encoding=options.encoding, basic=not options.items) \
encoding=options.encoding) \
.open(url, timeout=TIMEOUT * 2)
xml = con.read()