pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: drop 'basic' toggle 

Can't even remember the use case
master
pictuga 2020-04-05 16:03:06 +02:00
parent e6811138fd
commit 8a4d68d72c
2 changed files with 3 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
morss/crawler.py
View File

@ -33,7 +33,7 @@ MIMETYPE = {
DEFAULT_UA = 'Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0' DEFAULT_UA = 'Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0'
def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=False): def custom_handler(accept=None, strict=False, delay=None, encoding=None):
handlers = [] handlers = []
# as per urllib2 source code, these Handelers are added first # as per urllib2 source code, these Handelers are added first
@ -51,8 +51,6 @@ def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=F
handlers.append(HTTPEquivHandler()) handlers.append(HTTPEquivHandler())
handlers.append(HTTPRefreshHandler()) handlers.append(HTTPRefreshHandler())
handlers.append(UAHandler(DEFAULT_UA)) handlers.append(UAHandler(DEFAULT_UA))
if not basic:
handlers.append(AutoRefererHandler()) handlers.append(AutoRefererHandler())
handlers.append(EncodingFixHandler(encoding)) handlers.append(EncodingFixHandler(encoding))

2
morss/morss.py
View File

@ -336,7 +336,7 @@ def FeedFetch(url, options):
try: try:
con = crawler.custom_handler(accept='xml', strict=True, delay=delay, con = crawler.custom_handler(accept='xml', strict=True, delay=delay,
encoding=options.encoding, basic=not options.items) \ encoding=options.encoding) \
.open(url, timeout=TIMEOUT * 2) .open(url, timeout=TIMEOUT * 2)
xml = con.read() xml = con.read()