crawler: drop auto-referer

Was solving some issues. But creating even more issues.
2020-04-07 10:39:21 +02:00 · 2020-04-07 10:39:21 +02:00 · e5a82ff1f4
commit e5a82ff1f4
parent f3d1f92b39
1 changed files with 0 additions and 1 deletions
--- a/morss/crawler.py
+++ b/morss/crawler.py
@ -218,7 +218,6 @@ class BrowserlyHeaderHandler(BaseHandler):
    """ Add more headers to look less suspicious """

    def http_request(self, req):
-        req.add_unredirected_header('Referer', '%s://%s' % (req.type, req.host))
        req.add_unredirected_header('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
        req.add_unredirected_header('Accept-Language', 'en-US,en;q=0.5')
        return req