pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: add (unused) DebugHandler to output headers sent/received 

Saves a lot of time when debugging
master
pictuga 2017-10-27 23:10:03 +02:00
parent e0092387b1
commit fbe811384a
1 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
morss/crawler.py
View File

@ -44,6 +44,7 @@ def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=F
# FTPHandler, FileHandler, HTTPErrorProcessor] # FTPHandler, FileHandler, HTTPErrorProcessor]
# & HTTPSHandler # & HTTPSHandler
#handlers.append(DebugHandler())
handlers.append(HTTPCookieProcessor()) handlers.append(HTTPCookieProcessor())
handlers.append(GZIPHandler()) handlers.append(GZIPHandler())
handlers.append(HTTPEquivHandler()) handlers.append(HTTPEquivHandler())
@ -63,6 +64,21 @@ def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=F
return build_opener(*handlers) return build_opener(*handlers)
class DebugHandler(BaseHandler):
handler_order = 2000
def http_request(self, req):
print(repr(req.header_items()))
return req
def http_response(self, req, resp):
print(resp.headers.__dict__)
return resp
https_request = http_request
https_response = http_response
class GZIPHandler(BaseHandler): class GZIPHandler(BaseHandler):
def http_request(self, req): def http_request(self, req):
req.add_unredirected_header('Accept-Encoding', 'gzip') req.add_unredirected_header('Accept-Encoding', 'gzip')