Pick caching backend via env vars

master
pictuga 2020-08-23 18:43:18 +02:00
parent dcd3e4a675
commit 4dfebe78f7
4 changed files with 24 additions and 14 deletions

View File

@ -296,13 +296,15 @@ output = morss.FeedFormat(rss, options, 'unicode') # formats final feed
## Cache information ## Cache information
morss uses caching to make loading faster. There are 3 possible cache backends morss uses caching to make loading faster. There are 3 possible cache backends,
(visible in `morss/crawler.py`): which can be picked via environment variables:
- `{}`: a simple python in-memory dict() object - `(nothing/default)`: a simple python in-memory dict() object.
- `SQLiteCache`: sqlite3 cache. Default file location is in-memory (i.e. it will - `CACHE=sqlite`: sqlite3 cache. Default file location is in-memory (i.e. it
be cleared every time the program is run will be cleared every time the program is run). Path can be defined with
- `MySQLCacheHandler` `SQLITE_PATH`.
- `CACHE=mysql`: MySQL cache. Connection can be defined with the following
environment variables: `MYSQL_USER`, `MYSQL_PWD`, `MYSQL_DB`, `MYSQL_HOST`
## Configuration ## Configuration
### Length limitation ### Length limitation

View File

@ -101,8 +101,6 @@ def cgi_app(environ, start_response):
headers['content-type'] += '; charset=utf-8' headers['content-type'] += '; charset=utf-8'
crawler.default_cache = crawler.SQLiteCache(os.path.join(os.getcwd(), 'morss-cache.db'))
# get the work done # get the work done
url, rss = FeedFetch(url, options) url, rss = FeedFetch(url, options)

View File

@ -2,7 +2,6 @@ import sys
import os.path import os.path
import argparse import argparse
from . import crawler
from .morss import FeedFetch, FeedGather, FeedFormat from .morss import FeedFetch, FeedGather, FeedFormat
from .morss import Options from .morss import Options
@ -44,8 +43,6 @@ def cli_app():
options = Options(vars(parser.parse_args())) options = Options(vars(parser.parse_args()))
url = options.url url = options.url
crawler.default_cache = crawler.SQLiteCache(os.path.expanduser('~/.cache/morss-cache.db'))
url, rss = FeedFetch(url, options) url, rss = FeedFetch(url, options)
rss = FeedGather(rss, url, options) rss = FeedGather(rss, url, options)
out = FeedFormat(rss, options, 'unicode') out = FeedFormat(rss, options, 'unicode')

View File

@ -388,9 +388,6 @@ class HTTPRefreshHandler(BaseHandler):
https_response = http_response https_response = http_response
default_cache = {}
class CacheHandler(BaseHandler): class CacheHandler(BaseHandler):
" Cache based on etags/last-modified " " Cache based on etags/last-modified "
@ -659,6 +656,22 @@ class MySQLCacheHandler(BaseCache):
(url,) + value + value) (url,) + value + value)
if 'CACHE' in os.environ:
if os.environ['CACHE'] == 'mysql':
default_cache = MySQLCacheHandler(
user = os.getenv('MYSQL_USER'),
password = os.getenv('MYSQL_PWD'),
database = os.getenv('MYSQL_DB'),
host = os.getenv('MYSQL_HOST')
)
elif os.environ['CACHE'] == 'sqlite':
default_cache = SQLiteCache(os.getenv('SQLITE_PATH', ':memory:'))
else:
default_cache = {}
if __name__ == '__main__': if __name__ == '__main__':
req = adv_get(sys.argv[1] if len(sys.argv) > 1 else 'https://morss.it') req = adv_get(sys.argv[1] if len(sys.argv) > 1 else 'https://morss.it')