pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: ignore ssl via env var

master
pictuga 2020-10-03 19:57:08 +02:00
parent 8abd951d40
commit bd0bca69fc
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -345,6 +345,7 @@ Generic:
- `DEBUG=1`: to have some feedback from the script - `DEBUG=1`: to have some feedback from the script
execution. Useful for debugging. execution. Useful for debugging.
- `IGNORE_SSL=1`: to ignore SSL certs when fetch feeds and articles
- `DELAY` sets the browser cache delay, only for HTTP clients - `DELAY` sets the browser cache delay, only for HTTP clients
- `TIMEOUT` sets the HTTP timeout when fetching rss feeds and articles - `TIMEOUT` sets the HTTP timeout when fetching rss feeds and articles

9
morss/crawler.py
View File

@ -53,10 +53,6 @@ except NameError:
CACHE_SIZE = int(os.getenv('CACHE_SIZE', 1000)) # max number of items in cache (default: 1k items) CACHE_SIZE = int(os.getenv('CACHE_SIZE', 1000)) # max number of items in cache (default: 1k items)
CACHE_LIFESPAN = int(os.getenv('CACHE_LIFESPAN', 60)) # how often to auto-clear the cache (default: 1min) CACHE_LIFESPAN = int(os.getenv('CACHE_LIFESPAN', 60)) # how often to auto-clear the cache (default: 1min)
# uncomment the lines below to ignore SSL certs
#import ssl
#ssl._create_default_https_context = ssl._create_unverified_context
MIMETYPE = { MIMETYPE = {
'xml': ['text/xml', 'application/xml', 'application/rss+xml', 'application/rdf+xml', 'application/atom+xml', 'application/xhtml+xml'], 'xml': ['text/xml', 'application/xml', 'application/rss+xml', 'application/rdf+xml', 'application/atom+xml', 'application/xhtml+xml'],
@ -743,6 +739,11 @@ else:
default_cache = CappedDict() default_cache = CappedDict()
if 'IGNORE_SSL' in os.environ:
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
if __name__ == '__main__': if __name__ == '__main__':
req = adv_get(sys.argv[1] if len(sys.argv) > 1 else 'https://morss.it') req = adv_get(sys.argv[1] if len(sys.argv) > 1 else 'https://morss.it')