Fix bad characters detection

Now works with any encoding, no longer restricted to utf-8. Uses regex to find encoding (not perfect, but rather fast, since it's used on a substring)
master
pictuga 2013-09-15 14:57:37 +02:00
parent 3ba74649f6
commit 3176c2a8e8
1 changed files with 8 additions and 1 deletions

View File

@ -42,10 +42,17 @@ class FeedException(Exception):
pass
def parse(data):
data = data.decode('utf-8', 'replace').encode('utf-8')
# encoding
match = re.search('encoding=["\']?([0-9a-zA-Z-]+)', data[:100])
if match:
enc = match.groups()[0].lower()
data = data.decode(enc, 'ignore').encode(enc)
# parse
parser = etree.XMLParser(recover=True)
doc = etree.fromstring(data, parser)
# rss
match = doc.xpath("//atom03:feed|//atom:feed|//channel|//rdf:rdf|//rdf:RDF", namespaces=NSMAP)
if len(match):
mtable = { 'rdf:rdf': FeedParserRSS, 'channel': FeedParserRSS,