pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: drop auto-referer 

Was solving some issues. But creating even more issues.
master
pictuga 2020-04-07 10:39:21 +02:00
parent f3d1f92b39
commit e5a82ff1f4
1 changed files with 0 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
morss/crawler.py
View File

@ -218,7 +218,6 @@ class BrowserlyHeaderHandler(BaseHandler):
""" Add more headers to look less suspicious """ """ Add more headers to look less suspicious """
def http_request(self, req): def http_request(self, req):
req.add_unredirected_header('Referer', '%s://%s' % (req.type, req.host))
req.add_unredirected_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8') req.add_unredirected_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
req.add_unredirected_header('Accept-Language', 'en-US,en;q=0.5') req.add_unredirected_header('Accept-Language', 'en-US,en;q=0.5')
return req return req