pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Clean up sqlite code
default / test-lint (push) Successful in 1m26s Details
default / python-publish (push) Successful in 30s Details
default / docker-publish-deploy (push) Successful in 1m35s Details

master
pictuga 2023-06-26 01:30:47 +02:00
parent f4da40fffb
commit 6529fdbdd8
3 changed files with 7 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -81,9 +81,9 @@ From git
pip install git+https://git.pictuga.com/pictuga/morss.git#egg=morss[full] pip install git+https://git.pictuga.com/pictuga/morss.git#egg=morss[full]
``` ```
The full install includes all the cache backends. Otherwise, only in-memory and The full install includes all the cache backends. Otherwise, only in-memory
sqlite3 caches are available. The full install also includes gunicorn (for more cache is available. The full install also includes gunicorn (for more efficient
efficient HTTP handling). HTTP handling).
The dependency `lxml` is fairly long to install (especially on Raspberry Pi, as The dependency `lxml` is fairly long to install (especially on Raspberry Pi, as
C code needs to be compiled). If possible on your distribution, try installing C code needs to be compiled). If possible on your distribution, try installing
@ -353,7 +353,7 @@ Using cache and passing arguments:
```python ```python
>>> import morss >>> import morss
>>> url = 'http://feeds.bbci.co.uk/news/rss.xml' >>> url = 'http://feeds.bbci.co.uk/news/rss.xml'
>>> cache = '/tmp/morss-cache.db' # sqlite cache location >>> cache = '/tmp/morss-cache' # diskcache cache location
>>> options = {'csv':True} >>> options = {'csv':True}
>>> xml_string = morss.process(url, cache, options) >>> xml_string = morss.process(url, cache, options)
>>> xml_string[:50] >>> xml_string[:50]
@ -367,11 +367,10 @@ under the hood.
Doing it step-by-step: Doing it step-by-step:
```python ```python
import morss, morss.crawler import morss
url = 'http://newspaper.example/feed.xml' url = 'http://newspaper.example/feed.xml'
options = morss.Options(csv=True) # arguments options = morss.Options(csv=True) # arguments
morss.crawler.sqlite_default = '/tmp/morss-cache.db' # sqlite cache location
url, rss = morss.FeedFetch(url, options) # this only grabs the RSS feed url, rss = morss.FeedFetch(url, options) # this only grabs the RSS feed
rss = morss.FeedGather(rss, url, options) # this fills the feed and cleans it up rss = morss.FeedGather(rss, url, options) # this fills the feed and cleans it up

15
morss/caching.py
View File

@ -104,20 +104,7 @@ class DiskCacheHandler(BaseCache):
if 'CACHE' in os.environ: if 'CACHE' in os.environ:
if os.environ['CACHE'] == 'mysql': if os.environ['CACHE'] == 'redis':
default_cache = MySQLCacheHandler(
user = os.getenv('MYSQL_USER'),
password = os.getenv('MYSQL_PWD'),
database = os.getenv('MYSQL_DB'),
host = os.getenv('MYSQL_HOST', 'localhost')
)
elif os.environ['CACHE'] == 'sqlite':
default_cache = SQLiteCache(
os.getenv('SQLITE_PATH', ':memory:')
)
elif os.environ['CACHE'] == 'redis':
default_cache = RedisCacheHandler( default_cache = RedisCacheHandler(
host = os.getenv('REDIS_HOST', 'localhost'), host = os.getenv('REDIS_HOST', 'localhost'),
port = int(os.getenv('REDIS_PORT', 6379)), port = int(os.getenv('REDIS_PORT', 6379)),

2
morss/morss.py
View File

@ -428,7 +428,7 @@ def process(url, cache=None, options=None):
options = Options(options) options = Options(options)
if cache: if cache:
caching.default_cache = caching.SQLiteCache(cache) caching.default_cache = caching.DiskCacheHandler(cache)
url, rss = FeedFetch(url, options) url, rss = FeedFetch(url, options)
rss = FeedGather(rss, url, options) rss = FeedGather(rss, url, options)