More changes for JSOC

sunpy · Apr 18, 2023 · eff0b39 · eff0b39
1 parent 8cd66ee
commit eff0b39
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 9 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -17,7 +17,7 @@ repos:
           ]
         exclude: ".*(.fits|.fts|.fit|.header|.txt|tca.*|extern.*|.rst|.md|cm/__init__.py|sunpy/extern|sunpy/visualization/colormaps/color_tables.py)$"
   - repo: https://github.com/PyCQA/autoflake
-    rev: v2.0.2
+    rev: v2.1.1
     hooks:
       - id: autoflake
         args:

diff --git a/sunpy/net/dataretriever/sources/tests/test_goes_ud.py b/sunpy/net/dataretriever/sources/tests/test_goes_ud.py
@@ -32,7 +32,7 @@ def LCClient():
       'https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/'
       'goes16/l2/data/xrsf-l2-flx1s_science/2020/08/sci_xrsf-l2-flx1s_g16_d20200802_v2-2-0.nc',
       'https://data.ngdc.noaa.gov/platforms/solar-space-observing-satellites/goes/'
-      'goes16/l2/data/xrsf-l2-flx1s_science/2020/08/sci_xrsf-l2-flx1s_g16_d20200804_v2-2-0.nc')])
+      'goes17/l2/data/xrsf-l2-flx1s_science/2020/08/sci_xrsf-l2-flx1s_g17_d20200804_v2-2-0.nc')])
 def test_get_url_for_time_range(LCClient, timerange, url_start, url_end):
     qresponse = LCClient.search(timerange)
     urls = [i['url'] for i in qresponse]

diff --git a/sunpy/net/fido_factory.py b/sunpy/net/fido_factory.py
@@ -14,6 +14,7 @@
 
 import numpy as np
 import parfive
+from packaging.version import Version
 
 from astropy.table import Table
 
@@ -26,6 +27,8 @@
 
 __all__ = ['Fido', 'UnifiedResponse', 'UnifiedDownloaderFactory']
 
+parfive_version = Version(parfive.__version__)
+
 
 class UnifiedResponse(Sequence):
     """
@@ -381,7 +384,6 @@ def fetch(self, *query_results, path=None, max_conn=5, progress=True,
         if "wait" in kwargs:
             raise ValueError("wait is not a valid keyword argument to Fido.fetch.")
 
-        # TODO: Remove when parfive allows us to special case URLS.
         # Avoid more than one connection for JSOC only requests.
         from sunpy.net.jsoc import JSOCClient
 
@@ -396,7 +398,10 @@ def fetch(self, *query_results, path=None, max_conn=5, progress=True,
             if is_jsoc_only:
                 max_conn = 1
                 max_splits = 1
-            downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite, max_splits=max_splits)
+            if parfive_version >= Version("2.0.0"):
+                downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite, max_splits=max_splits)
+            else:
+                downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite)
         elif not isinstance(downloader, parfive.Downloader):
             raise TypeError("The downloader argument must be a parfive.Downloader instance.")
 
@@ -431,8 +436,7 @@ def fetch(self, *query_results, path=None, max_conn=5, progress=True,
                     reslist.append(result)
 
         results = downloader.download()
-        # Combine the results objects from all the clients into one Results
-        # object.
+        # Combine the results objects from all the clients into one Results object.
         for result in reslist:
             if not isinstance(result, Results):
                 raise TypeError(

diff --git a/sunpy/net/jsoc/jsoc.py b/sunpy/net/jsoc/jsoc.py
@@ -8,6 +8,7 @@
 import drms
 import numpy as np
 import parfive
+from packaging.version import Version
 
 import astropy.table
 import astropy.time
@@ -26,6 +27,7 @@
 
 PRIMEKEY_LIST_TIME = {'T_START', 'T_REC', 'T_OBS', 'MidTime', 'OBS_DATE',
                       'obsdate', 'DATE_OBS', 'starttime', 'stoptime', 'UTC_StartTime'}
+parfive_version = Version(parfive.__version__)
 
 
 class NotExportedError(Exception):
@@ -532,7 +534,10 @@ def get_request(self, requests, path=None, overwrite=False, progress=True,
             # Private communication from JSOC say we should not use more than one connection.
             if max_conn != self.default_max_conn:
                 log.info(f"Setting max parallel downloads to 1 for the JSOC client.")
-            downloader = Downloader(progress=progress, overwrite=overwrite, max_conn=1, max_splits=max_splits)
+            if parfive_version >= Version("2.0.0"):
+                downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite, max_splits=max_splits)
+            else:
+                downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite)
 
         urls = []
         for request in requests:
@@ -549,7 +554,7 @@ def get_request(self, requests, path=None, overwrite=False, progress=True,
                 print_message = "{0} URLs found for download. Full request totalling {1}MB"
                 print(print_message.format(len(urls), request._d['size']))
             for aurl, fname in zip(urls, paths):
-                downloader.enqueue_file(aurl, filename=fname)
+                downloader.enqueue_file(aurl, filename=fname, max_splits=max_splits)
 
         if dl_set and not wait:
             return Results()

diff --git a/sunpy/util/parfive_helpers.py b/sunpy/util/parfive_helpers.py
@@ -25,7 +25,6 @@ class Downloader(parfive.Downloader):
         def __init__(self, *args, **kwargs):
             headers = kwargs.pop("headers", {})
             kwargs["headers"] = {**sunpy_headers, **headers}
-
             # Works with conftest to hide the progress bar.
             progress = os.environ.get("PARFIVE_HIDE_PROGESS", None)
             if progress: