Various small bug fixes
parent
d01b943597
commit
ec8edb02f1
|
@ -255,9 +255,10 @@ output = morss.Format(rss, options) # formats final feed
|
||||||
|
|
||||||
## Cache information
|
## Cache information
|
||||||
|
|
||||||
morss uses caching to make loading faster. There are 2 possible cache backends
|
morss uses caching to make loading faster. There are 3 possible cache backends
|
||||||
(visible in `morss/crawler.py`):
|
(visible in `morss/crawler.py`):
|
||||||
|
|
||||||
|
- `{}`: a simple python in-memory dict() object
|
||||||
- `SQLiteCache`: sqlite3 cache. Default file location is in-memory (i.e. it will
|
- `SQLiteCache`: sqlite3 cache. Default file location is in-memory (i.e. it will
|
||||||
be cleared every time the program is run
|
be cleared every time the program is run
|
||||||
- `MySQLCacheHandler`
|
- `MySQLCacheHandler`
|
||||||
|
|
|
@ -465,6 +465,8 @@ class CacheHandler(BaseHandler):
|
||||||
|
|
||||||
|
|
||||||
class BaseCache:
|
class BaseCache:
|
||||||
|
""" Subclasses must behave like a dict """
|
||||||
|
|
||||||
def __contains__(self, url):
|
def __contains__(self, url):
|
||||||
try:
|
try:
|
||||||
self[url]
|
self[url]
|
||||||
|
|
|
@ -102,7 +102,7 @@ item_link = ./a/@href
|
||||||
item_desc = ./div[class=desc]
|
item_desc = ./div[class=desc]
|
||||||
item_content = ./div[class=content]
|
item_content = ./div[class=content]
|
||||||
|
|
||||||
base = file:www/sheet.xsl
|
base = file:sheet.xsl
|
||||||
|
|
||||||
[twitter]
|
[twitter]
|
||||||
mode = html
|
mode = html
|
||||||
|
|
|
@ -136,7 +136,7 @@ def ItemFix(item, feedurl='/'):
|
||||||
""" Improves feed items (absolute links, resolve feedburner links, etc) """
|
""" Improves feed items (absolute links, resolve feedburner links, etc) """
|
||||||
|
|
||||||
# check unwanted uppercase title
|
# check unwanted uppercase title
|
||||||
if len(item.title) > 20 and item.title.isupper():
|
if item.title is not None and len(item.title) > 20 and item.title.isupper():
|
||||||
item.title = item.title.title()
|
item.title = item.title.title()
|
||||||
|
|
||||||
# check if it includes link
|
# check if it includes link
|
||||||
|
@ -199,7 +199,7 @@ def ItemFix(item, feedurl='/'):
|
||||||
|
|
||||||
# reddit
|
# reddit
|
||||||
if urlparse(feedurl).netloc == 'www.reddit.com':
|
if urlparse(feedurl).netloc == 'www.reddit.com':
|
||||||
match = lxml.html.fromstring(item.desc).xpath('//a[text()="[link]"]/@href')
|
match = lxml.html.fromstring(item.content).xpath('//a[text()="[link]"]/@href')
|
||||||
if len(match):
|
if len(match):
|
||||||
item.link = match[0]
|
item.link = match[0]
|
||||||
log(item.link)
|
log(item.link)
|
||||||
|
|
|
@ -137,7 +137,7 @@ def score_all(node):
|
||||||
|
|
||||||
for child in node:
|
for child in node:
|
||||||
score = score_node(child)
|
score = score_node(child)
|
||||||
child.attrib['seen'] = 'yes, ' + str(int(score))
|
child.attrib['morss_own_score'] = str(float(score))
|
||||||
|
|
||||||
if score > 0 or len(list(child.iterancestors())) <= 2:
|
if score > 0 or len(list(child.iterancestors())) <= 2:
|
||||||
spread_score(child, score)
|
spread_score(child, score)
|
||||||
|
|
Loading…
Reference in New Issue