pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

readabilite: specify in- and out-going encoding

master
pictuga 2020-04-28 14:44:35 +02:00
parent 4a9b505499
commit 44a3e0edc4
2 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
morss/morss.py
View File

@ -260,7 +260,7 @@ def ItemFill(item, options, feedurl='/', fast=False):
log('non-text page') log('non-text page')
return True return True
out = readabilite.get_article(data, url=con.geturl(), encoding=encoding) out = readabilite.get_article(data, url=con.geturl(), encoding_in=encoding, encoding_out='unicode')
if out is not None: if out is not None:
item.content = out item.content = out
@ -635,7 +635,7 @@ def cgi_get(environ, start_response):
output = lxml.etree.tostring(html.getroottree(), encoding='utf-8') output = lxml.etree.tostring(html.getroottree(), encoding='utf-8')
elif options.get == 'article': elif options.get == 'article':
output = readabilite.get_article(data, url=con.geturl(), encoding=encoding, debug=options.debug) output = readabilite.get_article(data, url=con.geturl(), encoding_in=encoding, encoding_out='utf-8', debug=options.debug)
else: else:
raise MorssException('no :get option passed') raise MorssException('no :get option passed')

10
morss/readabilite.py
View File

@ -316,10 +316,10 @@ def get_best_node(ranked_grades):
return lowest return lowest
def get_article(data, url=None, encoding=None, debug=False, threshold=5): def get_article(data, url=None, encoding_in=None, encoding_out='unicode', debug=False, threshold=5):
" Input a raw html string, returns a raw html string of the article " " Input a raw html string, returns a raw html string of the article "
html = parse(data, encoding) html = parse(data, encoding_in)
score_all(html) score_all(html)
scores = rank_grades(get_all_scores(html)) scores = rank_grades(get_all_scores(html))
@ -341,7 +341,7 @@ def get_article(data, url=None, encoding=None, debug=False, threshold=5):
if url: if url:
best.make_links_absolute(url) best.make_links_absolute(url)
return lxml.etree.tostring(best if not debug else html, pretty_print=True) return lxml.etree.tostring(best if not debug else html, pretty_print=True, encoding=encoding_out)
if __name__ == '__main__': if __name__ == '__main__':
@ -349,7 +349,7 @@ if __name__ == '__main__':
from . import crawler from . import crawler
data, con, contenttype, encoding = crawler.adv_get(sys.argv[1] if len(sys.argv) > 1 else 'https://morss.it') data, con, contenttype, encoding = crawler.adv_get(sys.argv[1] if len(sys.argv) > 1 else 'https://morss.it')
article = get_article(data, url=con.geturl(), encoding=encoding) article = get_article(data, url=con.geturl(), encoding_in=encoding, encoding_out='unicode')
if not sys.flags.interactive: if not sys.flags.interactive:
print(article.decode(encoding)) print(article)