From 4dfebe78f7a4348dad0584edc193ad2e6f0839e0 Mon Sep 17 00:00:00 2001 From: pictuga Date: Sun, 23 Aug 2020 18:43:18 +0200 Subject: [PATCH] Pick caching backend via env vars --- README.md | 14 ++++++++------ morss/cgi.py | 2 -- morss/cli.py | 3 --- morss/crawler.py | 19 ++++++++++++++++--- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 915f7b4..a325e87 100644 --- a/README.md +++ b/README.md @@ -296,13 +296,15 @@ output = morss.FeedFormat(rss, options, 'unicode') # formats final feed ## Cache information -morss uses caching to make loading faster. There are 3 possible cache backends -(visible in `morss/crawler.py`): +morss uses caching to make loading faster. There are 3 possible cache backends, +which can be picked via environment variables: -- `{}`: a simple python in-memory dict() object -- `SQLiteCache`: sqlite3 cache. Default file location is in-memory (i.e. it will -be cleared every time the program is run -- `MySQLCacheHandler` +- `(nothing/default)`: a simple python in-memory dict() object. +- `CACHE=sqlite`: sqlite3 cache. Default file location is in-memory (i.e. it +will be cleared every time the program is run). Path can be defined with +`SQLITE_PATH`. +- `CACHE=mysql`: MySQL cache. Connection can be defined with the following +environment variables: `MYSQL_USER`, `MYSQL_PWD`, `MYSQL_DB`, `MYSQL_HOST` ## Configuration ### Length limitation diff --git a/morss/cgi.py b/morss/cgi.py index 2c22470..f1a8003 100644 --- a/morss/cgi.py +++ b/morss/cgi.py @@ -101,8 +101,6 @@ def cgi_app(environ, start_response): headers['content-type'] += '; charset=utf-8' - crawler.default_cache = crawler.SQLiteCache(os.path.join(os.getcwd(), 'morss-cache.db')) - # get the work done url, rss = FeedFetch(url, options) diff --git a/morss/cli.py b/morss/cli.py index bacd964..a58b800 100644 --- a/morss/cli.py +++ b/morss/cli.py @@ -2,7 +2,6 @@ import sys import os.path import argparse -from . import crawler from .morss import FeedFetch, FeedGather, FeedFormat from .morss import Options @@ -44,8 +43,6 @@ def cli_app(): options = Options(vars(parser.parse_args())) url = options.url - crawler.default_cache = crawler.SQLiteCache(os.path.expanduser('~/.cache/morss-cache.db')) - url, rss = FeedFetch(url, options) rss = FeedGather(rss, url, options) out = FeedFormat(rss, options, 'unicode') diff --git a/morss/crawler.py b/morss/crawler.py index d98fb1c..0573cc6 100644 --- a/morss/crawler.py +++ b/morss/crawler.py @@ -388,9 +388,6 @@ class HTTPRefreshHandler(BaseHandler): https_response = http_response -default_cache = {} - - class CacheHandler(BaseHandler): " Cache based on etags/last-modified " @@ -659,6 +656,22 @@ class MySQLCacheHandler(BaseCache): (url,) + value + value) +if 'CACHE' in os.environ: + if os.environ['CACHE'] == 'mysql': + default_cache = MySQLCacheHandler( + user = os.getenv('MYSQL_USER'), + password = os.getenv('MYSQL_PWD'), + database = os.getenv('MYSQL_DB'), + host = os.getenv('MYSQL_HOST') + ) + + elif os.environ['CACHE'] == 'sqlite': + default_cache = SQLiteCache(os.getenv('SQLITE_PATH', ':memory:')) + +else: + default_cache = {} + + if __name__ == '__main__': req = adv_get(sys.argv[1] if len(sys.argv) > 1 else 'https://morss.it')