pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

readabilite: clean up code

master
pictuga 2020-07-15 00:49:34 +02:00
parent c628ee802c
commit c6d3a0eb53
1 changed files with 16 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
morss/readabilite.py
View File

@ -47,12 +47,6 @@ def count_content(node):
return count_words(node.text_content()) + len(node.findall('.//img')) return count_words(node.text_content()) + len(node.findall('.//img'))
def percentile(N, P):
# https://stackoverflow.com/a/7464107
n = max(int(round(P * len(N) + 0.5)), 2)
return N[n-2]
class_bad = ['comment', 'community', 'extra', 'foot', class_bad = ['comment', 'community', 'extra', 'foot',
'sponsor', 'pagination', 'pager', 'tweet', 'twitter', 'com-', 'masthead', 'sponsor', 'pagination', 'pager', 'tweet', 'twitter', 'com-', 'masthead',
'media', 'meta', 'related', 'shopping', 'tags', 'tool', 'author', 'about', 'media', 'meta', 'related', 'shopping', 'tags', 'tool', 'author', 'about',
@ -198,8 +192,8 @@ def clean_node(node, keep_threshold=None):
parent.remove(node) parent.remove(node)
return return
if keep_threshold is not None and get_score(node) >= keep_threshold:
# high score, so keep # high score, so keep
if keep_threshold is not None and get_score(node) >= keep_threshold:
return return
gdparent = parent.getparent() gdparent = parent.getparent()
@ -300,44 +294,39 @@ def lowest_common_ancestor(nodeA, nodeB, max_depth=None):
return nodeA # should always find one tho, at least <html/>, but needed for max_depth return nodeA # should always find one tho, at least <html/>, but needed for max_depth
def rank_grades(grades):
# largest score to smallest
return sorted(grades.items(), key=lambda x: x[1], reverse=True)
def get_best_node(ranked_grades):
" To pick the best (raw) node. Another function will clean it "
if len(ranked_grades) == 1:
return ranked_grades[0]
lowest = lowest_common_ancestor(ranked_grades[0][0], ranked_grades[1][0], 3)
return lowest
def get_article(data, url=None, encoding_in=None, encoding_out='unicode', debug=False, threshold=5): def get_article(data, url=None, encoding_in=None, encoding_out='unicode', debug=False, threshold=5):
" Input a raw html string, returns a raw html string of the article " " Input a raw html string, returns a raw html string of the article "
html = parse(data, encoding_in) html = parse(data, encoding_in)
score_all(html) score_all(html)
scores = rank_grades(get_all_scores(html))
if not len(scores) or scores[0][1] < threshold: # rank all nodes (largest to smallest)
ranked_nodes = sorted(html.iter(), key=lambda x: get_score(x), reverse=True)
# minimum threshold
if not len(ranked_nodes) or get_score(ranked_nodes[0]) < threshold:
return None return None
best = get_best_node(scores) # take common ancestor or the two highest rated nodes
if len(ranked_nodes) > 1:
best = lowest_common_ancestor(ranked_nodes[0], ranked_nodes[1], 3)
else:
best = ranked_nodes[0]
# clean up
if not debug: if not debug:
keep_threshold = percentile([x[1] for x in scores], 0.1) keep_threshold = get_score(ranked_nodes[0]) * 3/4
clean_root(best, keep_threshold) clean_root(best, keep_threshold)
# check for spammy content (links only)
wc = count_words(best.text_content()) wc = count_words(best.text_content())
wca = count_words(' '.join([x.text_content() for x in best.findall('.//a')])) wca = count_words(' '.join([x.text_content() for x in best.findall('.//a')]))
if not debug and (wc - wca < 50 or float(wca) / wc > 0.3): if not debug and (wc - wca < 50 or float(wca) / wc > 0.3):
return None return None
# fix urls
if url: if url:
best.make_links_absolute(url) best.make_links_absolute(url)