pytest: first batch with test_feeds
continuous-integration/drone/push Build is failing
Details
continuous-integration/drone/push Build is failing
Details
parent
d6b90448f3
commit
c9ba0dc96b
|
|
@ -10,6 +10,7 @@ steps:
|
|||
- pip3 install --no-cache-dir .[full] .[dev]
|
||||
- isort --check-only --diff .
|
||||
- pylint morss --rcfile=.pylintrc --disable=C,R,W --fail-under=8
|
||||
- pytest --cov=morss tests
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
|
|
|
|||
|
|
@ -90,9 +90,6 @@ item_updated = updated
|
|||
[html]
|
||||
mode = html
|
||||
|
||||
path =
|
||||
http://localhost/
|
||||
|
||||
title = //div[@id='header']/h1
|
||||
desc = //div[@id='header']/p
|
||||
items = //div[@id='content']/div
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -23,7 +23,7 @@ setup(
|
|||
install_requires = ['lxml', 'bs4', 'python-dateutil', 'chardet'],
|
||||
extras_require = {
|
||||
'full': ['pymysql', 'redis', 'diskcache', 'gunicorn', 'setproctitle'],
|
||||
'dev': ['pylint']
|
||||
'dev': ['pylint', 'pytest', 'pytest-cov'],
|
||||
},
|
||||
python_requires = '>=2.7',
|
||||
package_data = {package_name: ['feedify.ini']},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
import os
|
||||
import os.path
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
try:
|
||||
# python2
|
||||
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
|
||||
from SimpleHTTPServer import SimpleHTTPRequestHandler
|
||||
except:
|
||||
# python3
|
||||
from http.server import (BaseHTTPRequestHandler, HTTPServer,
|
||||
SimpleHTTPRequestHandler)
|
||||
|
||||
class HTTPReplayHandler(SimpleHTTPRequestHandler):
|
||||
" Serves pages saved alongside with headers. See `curl --http1.1 -is http://...` "
|
||||
|
||||
directory = os.path.join(os.path.dirname(__file__), './samples/')
|
||||
|
||||
__init__ = BaseHTTPRequestHandler.__init__
|
||||
|
||||
def do_GET(self):
|
||||
path = self.translate_path(self.path)
|
||||
|
||||
if os.path.isdir(path):
|
||||
f = self.list_directory(path)
|
||||
|
||||
else:
|
||||
f = open(path, 'rb')
|
||||
|
||||
try:
|
||||
self.copyfile(f, self.wfile)
|
||||
|
||||
finally:
|
||||
f.close()
|
||||
|
||||
class MuteHTTPServer(HTTPServer):
|
||||
def handle_error(self, request, client_address):
|
||||
# mute errors
|
||||
pass
|
||||
|
||||
def make_server(port=8888):
|
||||
print('Serving http://localhost:%s/' % port)
|
||||
return MuteHTTPServer(('', port), RequestHandlerClass=HTTPReplayHandler)
|
||||
|
||||
@pytest.fixture
|
||||
def replay_server():
|
||||
httpd = make_server()
|
||||
thread = threading.Thread(target=httpd.serve_forever)
|
||||
thread.start()
|
||||
|
||||
yield
|
||||
|
||||
httpd.shutdown()
|
||||
thread.join()
|
||||
|
||||
if __name__ == '__main__':
|
||||
httpd = make_server()
|
||||
httpd.serve_forever()
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
HTTP/1.1 200 OK
|
||||
Content-Type: text/xml; charset=utf-8
|
||||
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title>!TITLE!</title>
|
||||
<subtitle>!DESC!</subtitle>
|
||||
<entry>
|
||||
<title>!ITEM_TITLE!</title>
|
||||
<summary>!ITEM_DESC!</summary>
|
||||
<content type="html">!ITEM_CONTENT!</content>
|
||||
<link href="!ITEM_LINK!"/>
|
||||
<updated>2022-01-01T00:00:01+01:00</updated>
|
||||
<published>2022-01-01T00:00:02+01:00</published>
|
||||
</entry>
|
||||
</feed>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
HTTP/1.1 200 OK
|
||||
content-type: application/xml
|
||||
|
||||
<?xml version='1.0' encoding='utf-8' ?>
|
||||
<feed version='0.3' xmlns='http://purl.org/atom/ns#'>
|
||||
<title>!TITLE!</title>
|
||||
<subtitle>!DESC!</subtitle>
|
||||
<entry>
|
||||
<title>!ITEM_TITLE!</title>
|
||||
<link rel='alternate' type='text/html' href='!ITEM_LINK!' />
|
||||
<summary>!ITEM_DESC!</summary>
|
||||
<content>!ITEM_CONTENT!</content>
|
||||
<issued>2022-01-01T00:00:01+01:00</issued> <!-- FIXME -->
|
||||
</entry>
|
||||
</feed>
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
HTTP/1.1 200 OK
|
||||
Content-Type: text/html; charset=utf-8
|
||||
|
||||
<html>
|
||||
<head></head>
|
||||
|
||||
<body>
|
||||
<div id="header">
|
||||
<h1>!TITLE!</h1>
|
||||
<p>!DESC!</p>
|
||||
</div>
|
||||
|
||||
<div id="content">
|
||||
<div class="item">
|
||||
<a target="_blank" href="!ITEM_LINK!">!ITEM_TITLE!</a>
|
||||
<div class="desc">!ITEM_DESC!</div>
|
||||
<div class="content">!ITEM_CONTENT!</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json; charset=utf-8
|
||||
|
||||
{
|
||||
"title": "!TITLE!",
|
||||
"desc": "!DESC!",
|
||||
"items": [
|
||||
{
|
||||
"title": "!ITEM_TITLE!",
|
||||
"time": "2022-01-01T00:00:01+0100",
|
||||
"url": "!ITEM_LINK!",
|
||||
"desc": "!ITEM_DESC!",
|
||||
"content": "!ITEM_CONTENT!"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
HTTP/1.1 200 OK
|
||||
Content-Type: text/xml; charset=utf-8
|
||||
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<rss xmlns:content="http://purl.org/rss/1.0/modules/content/" version="2.0">
|
||||
<channel>
|
||||
<title>!TITLE!</title>
|
||||
<description>!DESC!</description>
|
||||
<item>
|
||||
<title>!ITEM_TITLE!</title>
|
||||
<pubDate>Mon, 01 Jan 2022 00:00:01 +0100</pubDate>
|
||||
<link>!ITEM_LINK!</link>
|
||||
<description>!ITEM_DESC!</description>
|
||||
<content:encoded>!ITEM_CONTENT!</content:encoded>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
import pytest
|
||||
|
||||
from morss.crawler import adv_get
|
||||
from morss.feeds import *
|
||||
|
||||
|
||||
def get_feed(url):
|
||||
url = 'http://localhost:8888/%s' % url
|
||||
out = adv_get(url)
|
||||
feed = parse(out['data'], url=url, encoding=out['encoding'])
|
||||
return feed
|
||||
|
||||
def check_feed(feed):
|
||||
# NB. time and updated not covered
|
||||
assert feed.title == '!TITLE!'
|
||||
assert feed.desc == '!DESC!'
|
||||
assert feed[0] == feed.items[0]
|
||||
assert feed[0].title == '!ITEM_TITLE!'
|
||||
assert feed[0].link == '!ITEM_LINK!'
|
||||
assert '!ITEM_DESC!' in feed[0].desc # broader test due to possible inclusion of surrounding <div> in xml
|
||||
assert '!ITEM_CONTENT!' in feed[0].content
|
||||
|
||||
def check_output(feed):
|
||||
output = feed.tostring()
|
||||
assert '!TITLE!' in output
|
||||
assert '!DESC!' in output
|
||||
assert '!ITEM_TITLE!' in output
|
||||
assert '!ITEM_LINK!' in output
|
||||
assert '!ITEM_DESC!' in output
|
||||
assert '!ITEM_CONTENT!' in output
|
||||
|
||||
def check_change(feed):
|
||||
feed.title = '!TITLE2!'
|
||||
feed.desc = '!DESC2!'
|
||||
feed[0].title = '!ITEM_TITLE2!'
|
||||
feed[0].link = '!ITEM_LINK2!'
|
||||
feed[0].desc = '!ITEM_DESC2!'
|
||||
feed[0].content = '!ITEM_CONTENT2!'
|
||||
|
||||
assert feed.title == '!TITLE2!'
|
||||
assert feed.desc == '!DESC2!'
|
||||
assert feed[0].title == '!ITEM_TITLE2!'
|
||||
assert feed[0].link == '!ITEM_LINK2!'
|
||||
assert '!ITEM_DESC2!' in feed[0].desc
|
||||
assert '!ITEM_CONTENT2!' in feed[0].content
|
||||
|
||||
def check_add(feed):
|
||||
feed.append({
|
||||
'title': '!ITEM_TITLE3!',
|
||||
'link': '!ITEM_LINK3!',
|
||||
'desc': '!ITEM_DESC3!',
|
||||
'content': '!ITEM_CONTENT3!',
|
||||
})
|
||||
|
||||
assert feed[1].title == '!ITEM_TITLE3!'
|
||||
assert feed[1].link == '!ITEM_LINK3!'
|
||||
assert '!ITEM_DESC3!' in feed[1].desc
|
||||
assert '!ITEM_CONTENT3!' in feed[1].content
|
||||
|
||||
each_format = pytest.mark.parametrize('url', [
|
||||
'feed-rss-channel-utf-8.txt', 'feed-atom-utf-8.txt',
|
||||
'feed-atom03-utf-8.txt', 'feed-json-utf-8.txt', 'feed-html-utf-8.txt',
|
||||
])
|
||||
|
||||
each_check = pytest.mark.parametrize('check', [
|
||||
check_feed, check_output, check_change, check_add,
|
||||
])
|
||||
|
||||
@each_format
|
||||
@each_check
|
||||
def test_parse(replay_server, url, check):
|
||||
feed = get_feed(url)
|
||||
check(feed)
|
||||
|
||||
@each_format
|
||||
@each_check
|
||||
def test_convert_rss(replay_server, url, check):
|
||||
feed = get_feed(url)
|
||||
feed = feed.convert(FeedXML)
|
||||
check(feed)
|
||||
|
||||
@each_format
|
||||
@each_check
|
||||
def test_convert_json(replay_server, url, check):
|
||||
feed = get_feed(url)
|
||||
feed = feed.convert(FeedJSON)
|
||||
check(feed)
|
||||
|
||||
@each_format
|
||||
@each_check
|
||||
def test_convert_html(replay_server, url, check):
|
||||
feed = get_feed(url)
|
||||
feed = feed.convert(FeedHTML)
|
||||
if len(feed) > 1:
|
||||
# remove the 'blank' default html item
|
||||
del feed[0]
|
||||
check(feed)
|
||||
|
||||
@each_format
|
||||
def test_convert_csv(replay_server, url):
|
||||
# only csv output, not csv feed, check therefore differnet
|
||||
feed = get_feed(url)
|
||||
output = feed.tocsv()
|
||||
|
||||
assert '!ITEM_TITLE!' in output
|
||||
assert '!ITEM_LINK!' in output
|
||||
assert '!ITEM_DESC!' in output
|
||||
assert '!ITEM_CONTENT!' in output
|
||||
Loading…
Reference in New Issue