pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: catch html parsing errors

master
pictuga 2020-07-06 12:25:38 +02:00
parent 15951d228c
commit 64af86c11e
1 changed files with 21 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
morss/crawler.py
View File

@ -311,6 +311,8 @@ class AlternateHandler(BaseHandler):
# opps, not what we were looking for, let's see if the html page suggests an alternative page of the right types # opps, not what we were looking for, let's see if the html page suggests an alternative page of the right types
data = resp.read() data = resp.read()
try:
links = lxml.html.fromstring(data[:10000]).findall('.//link[@rel="alternate"]') links = lxml.html.fromstring(data[:10000]).findall('.//link[@rel="alternate"]')
for link in links: for link in links:
@ -320,6 +322,10 @@ class AlternateHandler(BaseHandler):
resp.headers['location'] = link.get('href') resp.headers['location'] = link.get('href')
break break
except (ValueError, SyntaxError):
# catch parsing errors
pass
fp = BytesIO(data) fp = BytesIO(data)
old_resp = resp old_resp = resp
resp = addinfourl(fp, old_resp.headers, old_resp.url, old_resp.code) resp = addinfourl(fp, old_resp.headers, old_resp.url, old_resp.code)
@ -340,11 +346,16 @@ class HTTPEquivHandler(BaseHandler):
if 200 <= resp.code < 300 and contenttype in MIMETYPE['html']: if 200 <= resp.code < 300 and contenttype in MIMETYPE['html']:
data = resp.read() data = resp.read()
try:
headers = lxml.html.fromstring(data[:10000]).findall('.//meta[@http-equiv]') headers = lxml.html.fromstring(data[:10000]).findall('.//meta[@http-equiv]')
for header in headers: for header in headers:
resp.headers[header.get('http-equiv').lower()] = header.get('content') resp.headers[header.get('http-equiv').lower()] = header.get('content')
except (ValueError, SyntaxError):
# catch parsing errors
pass
fp = BytesIO(data) fp = BytesIO(data)
old_resp = resp old_resp = resp
resp = addinfourl(fp, old_resp.headers, old_resp.url, old_resp.code) resp = addinfourl(fp, old_resp.headers, old_resp.url, old_resp.code)