pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

2to3: fix urllib header retrieval

master
pictuga 2015-03-11 01:03:16 +08:00
parent 6ae60d0343
commit cbeb01e555
2 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
morss/crawler.py
View File

@ -132,8 +132,8 @@ class GZIPHandler(BaseHandler):
def detect_encoding(data, con=None): def detect_encoding(data, con=None):
if con is not None and con.headers.getparam('charset'): if con is not None and con.info().get('charset'):
return con.headers.getparam('charset') return con.info().get('charset')
match = re.search('charset=["\']?([0-9a-zA-Z-]+)', data[:1000]) match = re.search('charset=["\']?([0-9a-zA-Z-]+)', data[:1000])
if match: if match:
@ -148,7 +148,7 @@ def detect_encoding(data, con=None):
class EncodingFixHandler(BaseHandler): class EncodingFixHandler(BaseHandler):
def http_response(self, req, resp): def http_response(self, req, resp):
maintype = resp.info()['Content-Type'].split('/')[0] maintype = resp.info().get('Content-Type', '').split('/')[0]
if 200 <= resp.code < 300 and maintype == 'text': if 200 <= resp.code < 300 and maintype == 'text':
data = resp.read() data = resp.read()
enc = detect_encoding(data, resp) enc = detect_encoding(data, resp)
@ -227,7 +227,7 @@ class ContentNegociationHandler(BaseHandler): #FIXME
class MetaRedirectHandler(BaseHandler): class MetaRedirectHandler(BaseHandler):
def http_response(self, req, resp): def http_response(self, req, resp):
contenttype = resp.info()['Content-Type'].split(';')[0] contenttype = resp.info().get('Content-Type', '').split(';')[0]
if 200 <= resp.code < 300 and contenttype.startswith('text/'): if 200 <= resp.code < 300 and contenttype.startswith('text/'):
if contenttype in MIMETYPE['html']: if contenttype in MIMETYPE['html']:
data = resp.read() data = resp.read()

8
morss/morss.py
View File

@ -391,7 +391,7 @@ def Fill(item, cache, options, feedurl='/', fast=False):
cache.set(link, 'error-http') cache.set(link, 'error-http')
return True return True
contenttype = con.info()['Content-Type'].split(';')[0] contenttype = con.info().get('Content-Type', '').split(';')[0]
if contenttype not in MIMETYPE['html'] and contenttype != 'text/plain': if contenttype not in MIMETYPE['html'] and contenttype != 'text/plain':
log('non-text page') log('non-text page')
cache.set(link, 'error-type') cache.set(link, 'error-type')
@ -457,10 +457,10 @@ def Fetch(url, cache, options):
raise MorssException('Error downloading feed') raise MorssException('Error downloading feed')
cache.set('xml', xml) cache.set('xml', xml)
cache.set('etag', con.headers.getheader('etag')) cache.set('etag', con.info().get('etag'))
cache.set('lastmodified', con.headers.getheader('last-modified')) cache.set('lastmodified', con.info().get('last-modified'))
contenttype = con.info()['Content-Type'].split(';')[0] contenttype = con.info().get('Content-Type', '').split(';')[0]
if url.startswith('https://itunes.apple.com/lookup?id='): if url.startswith('https://itunes.apple.com/lookup?id='):
style = 'itunes' style = 'itunes'