pictuga
/
morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Compare commits

base: pictuga:a2c4691090c5a8a6302a4234aad1d867b0f8c4a6
Branches Tags
pictuga:master
pictuga:v1.1
pictuga:v1.0
...
compare: pictuga:271ac8f80f5761da43f5345bec6a923b25677cf1
Branches Tags
pictuga:master
pictuga:v1.1
pictuga:v1.0

2 Commits
a2c4691090 ... 271ac8f80f

Author SHA1 Message Date
pictuga 271ac8f80f crawler: comment code a bit 2020-05-02 19:18:01 +02:00
pictuga 64e41b807d crawler: handle http:/ (single slash) 
Fixing one more corner case! malayalam.oneindia.com
 2020-05-02 19:17:15 +02:00
1 changed files with 7 additions and 1 deletions
Show Stats Expand all files Collapse all files

8
morss/crawler.py
View File

@ -123,15 +123,21 @@ def is_ascii(string):
def sanitize_url(url): def sanitize_url(url):
# make sure the url is unicode, i.e. not bytes
if isinstance(url, bytes): if isinstance(url, bytes):
url = url.decode() url = url.decode()
# make sure there's a protocol (http://)
if url.split(':', 1)[0] not in PROTOCOL: if url.split(':', 1)[0] not in PROTOCOL:
url = 'http://' + url url = 'http://' + url
# turns out some websites have really badly fomatted urls (fix http:/badurl)
url = re.sub('^(https?):/([^/])', r'\1://\2', url)
# escape spaces
url = url.replace(' ', '%20') url = url.replace(' ', '%20')
# Escape non-ascii unicode characters # escape non-ascii unicode characters
# https://stackoverflow.com/a/4391299 # https://stackoverflow.com/a/4391299
parts = list(urlparse(url)) parts = list(urlparse(url))