Skip to content

Commit

Permalink
unbreak api
Browse files Browse the repository at this point in the history
  • Loading branch information
nabobalis committed Mar 29, 2024
1 parent a1050be commit 0e03f22
Showing 1 changed file with 24 additions and 14 deletions.
38 changes: 24 additions & 14 deletions sunpy/data/data_manager/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,15 @@ def __init__(self, downloader, storage, cache_dir, expiry=10*u.day):
self._cache_dir = Path(cache_dir)
self._expiry = expiry if expiry is None else TimeDelta(expiry)

def download(self, url, namespace='', redownload=False):
def download(self, urls, namespace='', redownload=False):
"""
Downloads the files from the urls.
Parameters
----------
urls : `str`
A single url.
urls : `list` of path-like or one path-like
A list of urls or a single url.
The list is for urls of the same file but from different sources.
namespace : `str`, optional
A namespace to be used for the file name.
Defaults to an empty string.
Expand All @@ -56,13 +57,18 @@ def download(self, url, namespace='', redownload=False):
`pathlib.Path`
Path to the downloaded file.
"""
if isinstance(urls, str | Path):
urls = [urls]
# Logic flow
# 1. If redownload: Download, update cache and return file path
# 2. If not redownload: Check cache,
# i. If present in cache:
# - If cache expired, download and add to cache and remove entry from cache
# - If cache not expired, return path
present = self._get_by_url(url)
for url in urls:
present = self._get_by_url(url)
if present:
break
if present and not redownload and not self._has_expired(present):
return Path(present['file_path'])
try:
Expand Down Expand Up @@ -126,24 +132,28 @@ def _get_by_url(self, url):
"""
return self._storage.find_by_key('url', url)

def _download_and_hash(self, url, namespace=''):
def _download_and_hash(self, urls, namespace=''):
"""
Downloads the file and returns the path, hash and url it used to download.
Parameters
----------
urls : `str`
A url.
urls : `list` of `str`
List of urls.
Returns
-------
`str`, `str`, `str`
Path, hash and URL of the file.
"""
try:
path = self._cache_dir / (namespace + get_filename(urlopen(url), url))
self._downloader.download(url, path)
shahash = hash_file(path)
return path, shahash, url
except Exception as e:
raise RuntimeError(f"{e}") from e
errors = []
for url in urls:
try:
path = self._cache_dir / (namespace + get_filename(urlopen(url), url))
self._downloader.download(url, path)
shahash = hash_file(path)
return path, shahash, url
except Exception as e:
warn_user(f"{e}")
errors.append(f"{e}")
raise RuntimeError(errors)

0 comments on commit 0e03f22

Please sign in to comment.