Compare commits

..

No commits in common. "7c3091d64c796e20e64e423b1de6899d64866b50" and "961a31141f5353a76c4101a488f4a776171d8f9e" have entirely different histories.

1 changed files with 27 additions and 97 deletions

View File

@ -18,7 +18,6 @@ from . import readabilite
import wsgiref.simple_server import wsgiref.simple_server
import wsgiref.handlers import wsgiref.handlers
import cgitb
try: try:
@ -45,7 +44,7 @@ THREADS = 10 # number of threads (1 for single-threaded)
DEBUG = False DEBUG = False
PORT = 8080 PORT = 8080
PROTOCOL = ['http', 'https'] PROTOCOL = ['http', 'https', 'ftp']
def filterOptions(options): def filterOptions(options):
@ -67,7 +66,6 @@ def log(txt, force=False):
if DEBUG or force: if DEBUG or force:
if 'REQUEST_URI' in os.environ: if 'REQUEST_URI' in os.environ:
open('morss.log', 'a').write("%s\n" % repr(txt)) open('morss.log', 'a').write("%s\n" % repr(txt))
else: else:
print(repr(txt)) print(repr(txt))
@ -75,7 +73,6 @@ def log(txt, force=False):
def len_html(txt): def len_html(txt):
if len(txt): if len(txt):
return len(lxml.html.fromstring(txt).text_content()) return len(lxml.html.fromstring(txt).text_content())
else: else:
return 0 return 0
@ -83,7 +80,6 @@ def len_html(txt):
def count_words(txt): def count_words(txt):
if len(txt): if len(txt):
return len(lxml.html.fromstring(txt).text_content().split()) return len(lxml.html.fromstring(txt).text_content().split())
return 0 return 0
@ -92,14 +88,12 @@ class Options:
if len(args): if len(args):
self.options = args self.options = args
self.options.update(options or {}) self.options.update(options or {})
else: else:
self.options = options or {} self.options = options or {}
def __getattr__(self, key): def __getattr__(self, key):
if key in self.options: if key in self.options:
return self.options[key] return self.options[key]
else: else:
return False return False
@ -113,23 +107,17 @@ class Options:
def parseOptions(options): def parseOptions(options):
""" Turns ['md=True'] into {'md':True} """ """ Turns ['md=True'] into {'md':True} """
out = {} out = {}
for option in options: for option in options:
split = option.split('=', 1) split = option.split('=', 1)
if len(split) > 1: if len(split) > 1:
if split[0].lower() == 'true': if split[0].lower() == 'true':
out[split[0]] = True out[split[0]] = True
elif split[0].lower() == 'false': elif split[0].lower() == 'false':
out[split[0]] = False out[split[0]] = False
else: else:
out[split[0]] = split[1] out[split[0]] = split[1]
else: else:
out[split[0]] = True out[split[0]] = True
return out return out
@ -220,7 +208,6 @@ def ItemFill(item, options, feedurl='/', fast=False):
if len(match): if len(match):
link = match[0] link = match[0]
log(link) log(link)
else: else:
link = None link = None
@ -230,7 +217,6 @@ def ItemFill(item, options, feedurl='/', fast=False):
if len(match) and urlparse(match[0]).netloc != 'www.facebook.com': if len(match) and urlparse(match[0]).netloc != 'www.facebook.com':
link = match[0] link = match[0]
log(link) log(link)
else: else:
link = None link = None
@ -313,7 +299,6 @@ def UrlFix(url):
return url return url
def FeedFetch(url, options): def FeedFetch(url, options):
# allow for code execution for feedify # allow for code execution for feedify
pre = feedify.pre_worker(url) pre = feedify.pre_worker(url)
@ -342,6 +327,7 @@ def FeedFetch(url, options):
if options.items: if options.items:
# using custom rules # using custom rules
rss = feeds.FeedHTML(xml) rss = feeds.FeedHTML(xml)
feed.rule
rss.rules['items'] = options.items rss.rules['items'] = options.items
@ -354,8 +340,6 @@ def FeedFetch(url, options):
if options.item_time: if options.item_time:
rss.rules['item_time'] = options.item_time rss.rules['item_time'] = options.item_time
rss = rss.convert(feeds.FeedXML)
else: else:
try: try:
rss = feeds.parse(xml, url, contenttype) rss = feeds.parse(xml, url, contenttype)
@ -393,7 +377,6 @@ def FeedGather(rss, url, options):
value = queue.get() value = queue.get()
try: try:
worker(*value) worker(*value)
except Exception as e: except Exception as e:
log('Thread Error: %s' % e.message) log('Thread Error: %s' % e.message)
queue.task_done() queue.task_done()
@ -433,7 +416,6 @@ def FeedGather(rss, url, options):
for i, item in enumerate(list(rss.items)): for i, item in enumerate(list(rss.items)):
if threads == 1: if threads == 1:
worker(*[i, item]) worker(*[i, item])
else: else:
queue.put([i, item]) queue.put([i, item])
@ -453,38 +435,37 @@ def FeedGather(rss, url, options):
return rss return rss
def FeedFormat(rss, options, encoding='utf-8'): def FeedFormat(rss, options):
if options.callback: if options.callback:
if re.match(r'^[a-zA-Z0-9\.]+$', options.callback) is not None: if re.match(r'^[a-zA-Z0-9\.]+$', options.callback) is not None:
out = '%s(%s)' % (options.callback, rss.tojson(encoding='unicode')) return '%s(%s)' % (options.callback, rss.tojson())
return out if encoding == 'unicode' else out.encode(encoding)
else: else:
raise MorssException('Invalid callback var name') raise MorssException('Invalid callback var name')
elif options.json: elif options.json:
if options.indent: if options.indent:
return rss.tojson(encoding=encoding, indent=4) return rss.tojson(encoding='UTF-8', indent=4)
else: else:
return rss.tojson(encoding=encoding) return rss.tojson(encoding='UTF-8')
elif options.csv: elif options.csv:
return rss.tocsv(encoding=encoding) return rss.tocsv(encoding='UTF-8')
elif options.reader: elif options.reader:
if options.indent: if options.indent:
return rss.tohtml(encoding=encoding, pretty_print=True) return rss.tohtml(encoding='UTF-8', pretty_print=True)
else: else:
return rss.tohtml(encoding=encoding) return rss.tohtml(encoding='UTF-8')
else: else:
if options.indent: if options.indent:
return rss.torss(xml_declaration=True, encoding=encoding, pretty_print=True) return rss.torss(xml_declaration=True, encoding='UTF-8', pretty_print=True)
else: else:
return rss.torss(xml_declaration=True, encoding=encoding) return rss.torss(xml_declaration=True, encoding='UTF-8')
def process(url, cache=None, options=None): def process(url, cache=None, options=None):
@ -518,7 +499,7 @@ def cgi_app(environ, start_response):
if url.startswith(':'): if url.startswith(':'):
split = url.split('/', 1) split = url.split('/', 1)
raw_options = split[0].replace('|', '/').replace('\\\'', '\'').split(':')[1:] options = split[0].replace('|', '/').replace('\\\'', '\'').split(':')[1:]
if len(split) > 1: if len(split) > 1:
url = split[1] url = split[1]
@ -526,10 +507,10 @@ def cgi_app(environ, start_response):
url = '' url = ''
else: else:
raw_options = [] options = []
# init # init
options = Options(filterOptions(parseOptions(raw_options))) options = Options(filterOptions(parseOptions(options)))
headers = {} headers = {}
global DEBUG global DEBUG
@ -570,42 +551,18 @@ def cgi_app(environ, start_response):
rss = FeedGather(rss, url, options) rss = FeedGather(rss, url, options)
out = FeedFormat(rss, options) out = FeedFormat(rss, options)
if options.silent: if not options.silent:
return [''] return out
else:
return [out]
def middleware(func): def cgi_wrapper(environ, start_response):
" Decorator to turn a function into a wsgi middleware " # simple http server for html and css
# This is called when parsing the code
def app_builder(app):
# This is called when doing app = cgi_wrapper(app)
def app_wrap(environ, start_response):
# This is called when a http request is being processed
return func(environ, start_response, app)
return app_wrap
return app_builder
@middleware
def cgi_file_handler(environ, start_response, app):
" Simple HTTP server to serve static files (.html, .css, etc.) "
files = { files = {
'': 'text/html', '': 'text/html',
'index.html': 'text/html', 'index.html': 'text/html'}
'sheet.xsl': 'text/xsl'}
if 'REQUEST_URI' in environ: if 'REQUEST_URI' in environ:
url = environ['REQUEST_URI'][1:] url = environ['REQUEST_URI'][1:]
else: else:
url = environ['PATH_INFO'][1:] url = environ['PATH_INFO'][1:]
@ -634,29 +591,16 @@ def cgi_file_handler(environ, start_response, app):
start_response(headers['status'], list(headers.items())) start_response(headers['status'], list(headers.items()))
return ['Error %s' % headers['status']] return ['Error %s' % headers['status']]
else: # actual morss use
return app(environ, start_response)
@middleware
def cgi_error_handler(environ, start_response, app):
try: try:
return app(environ, start_response) return [cgi_app(environ, start_response) or '(empty)']
except (KeyboardInterrupt, SystemExit): except (KeyboardInterrupt, SystemExit):
raise raise
except Exception as e: except Exception as e:
headers = {'status': '500 Oops', 'content-type': 'text/html'} headers = {'status': '500 Oops', 'content-type': 'text/plain'}
start_response(headers['status'], list(headers.items()), sys.exc_info()) start_response(headers['status'], list(headers.items()), sys.exc_info())
log('ERROR: %s' % repr(e), force=True) log('ERROR <%s>: %s' % (url, e.message), force=True)
return [cgitb.html(sys.exc_info())] return ['An error happened:\n%s' % e.message]
@middleware
def cgi_encode(environ, start_response, app):
out = app(environ, start_response)
return [x if isinstance(x, bytes) else x.encode('utf-8') for x in out]
def cli_app(): def cli_app():
@ -683,7 +627,6 @@ def isInt(string):
try: try:
int(string) int(string)
return True return True
except ValueError: except ValueError:
return False return False
@ -691,12 +634,7 @@ def isInt(string):
def main(): def main():
if 'REQUEST_URI' in os.environ: if 'REQUEST_URI' in os.environ:
# mod_cgi # mod_cgi
wsgiref.handlers.CGIHandler().run(cgi_wrapper)
app = cgi_app
app = cgi_error_handler(app)
app = cgi_encode(app)
wsgiref.handlers.CGIHandler().run(app)
elif len(sys.argv) <= 1 or isInt(sys.argv[1]) or '--root' in sys.argv[1:]: elif len(sys.argv) <= 1 or isInt(sys.argv[1]) or '--root' in sys.argv[1:]:
# start internal (basic) http server # start internal (basic) http server
@ -705,30 +643,22 @@ def main():
argPort = int(sys.argv[1]) argPort = int(sys.argv[1])
if argPort > 0: if argPort > 0:
port = argPort port = argPort
else: else:
raise MorssException('Port must be positive integer') raise MorssException('Port must be positive integer')
else: else:
port = PORT port = PORT
app = cgi_app print('Serving http://localhost:%s/'%port)
app = cgi_file_handler(app) httpd = wsgiref.simple_server.make_server('', port, cgi_wrapper)
app = cgi_error_handler(app)
app = cgi_encode(app)
print('Serving http://localhost:%s/' % port)
httpd = wsgiref.simple_server.make_server('', port, app)
httpd.serve_forever() httpd.serve_forever()
else: else:
# as a CLI app # as a CLI app
try: try:
cli_app() cli_app()
except (KeyboardInterrupt, SystemExit): except (KeyboardInterrupt, SystemExit):
raise raise
except Exception as e: except Exception as e:
print('ERROR: %s' % e.message) print('ERROR: %s' % e.message)