pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

2to3: urllib mimetype

master
pictuga 2015-03-03 00:55:58 +08:00
parent 7bd448789d
commit dbb3883516
2 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
morss/crawler.py
View File

@ -148,7 +148,8 @@ def detect_encoding(data, con=None):
class EncodingFixHandler(BaseHandler): class EncodingFixHandler(BaseHandler):
def http_response(self, req, resp): def http_response(self, req, resp):
if 200 <= resp.code < 300 and resp.info().maintype == 'text': maintype = resp.info()['Content-Type'].split('/')[0]
if 200 <= resp.code < 300 and maintype == 'text':
data = resp.read() data = resp.read()
enc = detect_encoding(data, resp) enc = detect_encoding(data, resp)
@ -226,8 +227,9 @@ class ContentNegociationHandler(BaseHandler): #FIXME
class MetaRedirectHandler(BaseHandler): class MetaRedirectHandler(BaseHandler):
def http_response(self, req, resp): def http_response(self, req, resp):
if 200 <= resp.code < 300 and resp.info().maintype == 'text': contenttype = resp.info()['Content-Type'].split(';')[0]
if resp.info().type in MIMETYPE['html']: if 200 <= resp.code < 300 and contenttype.startswith('text/'):
if contenttype in MIMETYPE['html']:
data = resp.read() data = resp.read()
match = re.search(r'(?i)<meta http-equiv=.refresh[^>]*?url=(http.*?)["\']', data) match = re.search(r'(?i)<meta http-equiv=.refresh[^>]*?url=(http.*?)["\']', data)
if match: if match:

11
morss/morss.py
View File

@ -391,7 +391,8 @@ def Fill(item, cache, options, feedurl='/', fast=False):
cache.set(link, 'error-http') cache.set(link, 'error-http')
return True return True
if con.info().type not in MIMETYPE['html'] and con.info().type != 'text/plain': contenttype = con.info()['Content-Type'].split(';')[0]
if contenttype not in MIMETYPE['html'] and contenttype != 'text/plain':
log('non-text page') log('non-text page')
cache.set(link, 'error-type') cache.set(link, 'error-type')
return True return True
@ -459,17 +460,19 @@ def Fetch(url, cache, options):
cache.set('etag', con.headers.getheader('etag')) cache.set('etag', con.headers.getheader('etag'))
cache.set('lastmodified', con.headers.getheader('last-modified')) cache.set('lastmodified', con.headers.getheader('last-modified'))
contenttype = con.info()['Content-Type'].split(';')[0]
if url.startswith('https://itunes.apple.com/lookup?id='): if url.startswith('https://itunes.apple.com/lookup?id='):
style = 'itunes' style = 'itunes'
elif xml.startswith('<?xml') or con.info().type in MIMETYPE['xml']: elif xml.startswith('<?xml') or contenttype in MIMETYPE['xml']:
style = 'normal' style = 'normal'
elif feedify.supported(url): elif feedify.supported(url):
style = 'feedify' style = 'feedify'
elif con.info().type in MIMETYPE['html']: elif contenttype in MIMETYPE['html']:
style = 'html' style = 'html'
else: else:
style = 'none' style = 'none'
log(con.info().type) log(contenttype)
cache.set('style', style) cache.set('style', style)