Compare commits

..

3 Commits

Author SHA1 Message Date
pictuga 6529fdbdd8 Clean up sqlite code
default / test-lint (push) Successful in 1m26s Details
default / python-publish (push) Successful in 30s Details
default / docker-publish-deploy (push) Successful in 1m35s Details
2023-06-26 01:30:47 +02:00
pictuga f4da40fffb actions: fix deploy 2023-06-26 01:29:00 +02:00
pictuga d27fc93f75 actions: clean up 2023-06-26 01:28:33 +02:00
4 changed files with 36 additions and 47 deletions

View File

@ -8,64 +8,59 @@ jobs:
test-lint: test-lint:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v3 - name: Checkout
uses: actions/checkout@v3
with: with:
fetch-depth: 0 fetch-depth: 0
- run: apt-get -y update && apt-get -y install python3-pip libenchant-2-2 aspell-en
- run: pip3 install .[full] .[dev] - name: Prepare image
run: apt-get -y update && apt-get -y install python3-pip libenchant-2-2 aspell-en
- name: Install dependencies
run: pip3 install .[full] .[dev]
- run: isort --check-only --diff . - run: isort --check-only --diff .
- run: pylint morss --rcfile=.pylintrc --disable=C,R,W --fail-under=8 - run: pylint morss --rcfile=.pylintrc --disable=C,R,W --fail-under=8
- run: pytest --cov=morss tests - run: pytest --cov=morss tests
python-publish:
publish-deploy:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v3
with: with:
fetch-depth: 0 fetch-depth: 0
- run: apt-get -y update && apt-get -y install python3-pip python3-build
- run: python3 -m build - name: Prepare image
run: apt-get -y update && apt-get -y install python3-pip python3-build
- name: Build package
run: python3 -m build
- name: Publish package - name: Publish package
uses: https://github.com/pypa/gh-action-pypi-publish@release/v1 uses: https://github.com/pypa/gh-action-pypi-publish@release/v1
with: with:
password: ${{ secrets.pypi_api_token }} password: ${{ secrets.pypi_api_token }}
- name: Push python package to the server
uses: https://github.com/appleboy/scp-action@v0.1.4
with:
host: ${{ secrets.ssh_host }}
username: ${{ secrets.ssh_user }}
key: ${{ secrets.ssh_key }}
source: dist/morss-*.tar.gz
target: /home/ubuntu
- name: Install & reload the server
uses: https://github.com/appleboy/ssh-action@v0.1.10
with:
host: ${{ secrets.ssh_host }}
username: ${{ secrets.ssh_user }}
key: ${{ secrets.ssh_key }}
script: |
sudo pip install --upgrade dist/morss*-.tar.gz[full]
sudo rm -r dist
sudo morss-helper reload
docker-publish: docker-publish-deploy:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: container:
image: catthehacker/ubuntu:act-latest image: catthehacker/ubuntu:act-latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v3
- name: Set up QEMU - name: Set up QEMU
uses: https://github.com/docker/setup-qemu-action@v2 uses: https://github.com/docker/setup-qemu-action@v2
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: https://github.com/docker/setup-buildx-action@v2 uses: https://github.com/docker/setup-buildx-action@v2
- name: Login to Docker Hub - name: Login to Docker Hub
uses: https://github.com/docker/login-action@v2 uses: https://github.com/docker/login-action@v2
with: with:
username: ${{ secrets.docker_user }} username: ${{ secrets.docker_user }}
password: ${{ secrets.docker_pwd }} password: ${{ secrets.docker_pwd }}
- name: Build and push - name: Build and push
uses: https://github.com/docker/build-push-action@v4 uses: https://github.com/docker/build-push-action@v4
with: with:
@ -73,3 +68,11 @@ jobs:
platforms: linux/amd64,linux/arm64,linux/arm/v7 platforms: linux/amd64,linux/arm64,linux/arm/v7
push: true push: true
tags: ${{ secrets.docker_repo }} tags: ${{ secrets.docker_repo }}
- name: Deploy on server
uses: https://github.com/appleboy/ssh-action@v0.1.10
with:
host: ${{ secrets.ssh_host }}
username: ${{ secrets.ssh_user }}
key: ${{ secrets.ssh_key }}
script: morss-update

View File

@ -81,9 +81,9 @@ From git
pip install git+https://git.pictuga.com/pictuga/morss.git#egg=morss[full] pip install git+https://git.pictuga.com/pictuga/morss.git#egg=morss[full]
``` ```
The full install includes all the cache backends. Otherwise, only in-memory and The full install includes all the cache backends. Otherwise, only in-memory
sqlite3 caches are available. The full install also includes gunicorn (for more cache is available. The full install also includes gunicorn (for more efficient
efficient HTTP handling). HTTP handling).
The dependency `lxml` is fairly long to install (especially on Raspberry Pi, as The dependency `lxml` is fairly long to install (especially on Raspberry Pi, as
C code needs to be compiled). If possible on your distribution, try installing C code needs to be compiled). If possible on your distribution, try installing
@ -353,7 +353,7 @@ Using cache and passing arguments:
```python ```python
>>> import morss >>> import morss
>>> url = 'http://feeds.bbci.co.uk/news/rss.xml' >>> url = 'http://feeds.bbci.co.uk/news/rss.xml'
>>> cache = '/tmp/morss-cache.db' # sqlite cache location >>> cache = '/tmp/morss-cache' # diskcache cache location
>>> options = {'csv':True} >>> options = {'csv':True}
>>> xml_string = morss.process(url, cache, options) >>> xml_string = morss.process(url, cache, options)
>>> xml_string[:50] >>> xml_string[:50]
@ -367,11 +367,10 @@ under the hood.
Doing it step-by-step: Doing it step-by-step:
```python ```python
import morss, morss.crawler import morss
url = 'http://newspaper.example/feed.xml' url = 'http://newspaper.example/feed.xml'
options = morss.Options(csv=True) # arguments options = morss.Options(csv=True) # arguments
morss.crawler.sqlite_default = '/tmp/morss-cache.db' # sqlite cache location
url, rss = morss.FeedFetch(url, options) # this only grabs the RSS feed url, rss = morss.FeedFetch(url, options) # this only grabs the RSS feed
rss = morss.FeedGather(rss, url, options) # this fills the feed and cleans it up rss = morss.FeedGather(rss, url, options) # this fills the feed and cleans it up

View File

@ -104,20 +104,7 @@ class DiskCacheHandler(BaseCache):
if 'CACHE' in os.environ: if 'CACHE' in os.environ:
if os.environ['CACHE'] == 'mysql': if os.environ['CACHE'] == 'redis':
default_cache = MySQLCacheHandler(
user = os.getenv('MYSQL_USER'),
password = os.getenv('MYSQL_PWD'),
database = os.getenv('MYSQL_DB'),
host = os.getenv('MYSQL_HOST', 'localhost')
)
elif os.environ['CACHE'] == 'sqlite':
default_cache = SQLiteCache(
os.getenv('SQLITE_PATH', ':memory:')
)
elif os.environ['CACHE'] == 'redis':
default_cache = RedisCacheHandler( default_cache = RedisCacheHandler(
host = os.getenv('REDIS_HOST', 'localhost'), host = os.getenv('REDIS_HOST', 'localhost'),
port = int(os.getenv('REDIS_PORT', 6379)), port = int(os.getenv('REDIS_PORT', 6379)),

View File

@ -428,7 +428,7 @@ def process(url, cache=None, options=None):
options = Options(options) options = Options(options)
if cache: if cache:
caching.default_cache = caching.SQLiteCache(cache) caching.default_cache = caching.DiskCacheHandler(cache)
url, rss = FeedFetch(url, options) url, rss = FeedFetch(url, options)
rss = FeedGather(rss, url, options) rss = FeedGather(rss, url, options)