import os.path as osp
import ssl
import sys
import urllib
from typing import Optional
from .makedirs import makedirs
[docs]def download_url(url: str, folder: str, log: bool = True,
filename: Optional[str] = None):
r"""Downloads the content of an URL to a specific folder.
Args:
url (string): The url.
folder (string): The folder.
log (bool, optional): If :obj:`False`, will not print anything to the
console. (default: :obj:`True`)
"""
if filename is None:
filename = url.rpartition('/')[2]
filename = filename if filename[0] == '?' else filename.split('?')[0]
path = osp.join(folder, filename)
if osp.exists(path): # pragma: no cover
if log:
print(f'Using existing file {filename}', file=sys.stderr)
return path
if log:
print(f'Downloading {url}', file=sys.stderr)
makedirs(folder)
context = ssl._create_unverified_context()
data = urllib.request.urlopen(url, context=context)
with open(path, 'wb') as f:
# workaround for https://bugs.python.org/issue42853
while True:
chunk = data.read(10 * 1024 * 1024)
if not chunk:
break
f.write(chunk)
return path