ADAPT Client

.gitignore

@@ -281,11 +281,15 @@ package.json
 
 # Log files generated by 'vagrant up'
 *.log
+
 # asv stuff
 asv_env
 asv_results
 html
+
+# Figure tests
 results
 result_images
+
 # Save from Map.save example
 aia171.fits

changelog/7463.feature.rst

@@ -0,0 +1 @@
+A new client (`sunpy.net.dataretriever.ADAPTClient`) has been added to search and download `ADAPT <https://gong.nso.edu/adapt/maps/gong/>`__ files.

docs/reference/net.rst

@@ -39,6 +39,9 @@ Dataretriever
 .. automodapi:: sunpy.net.dataretriever.attrs.goes
    :headings: ^"
 
+.. automodapi:: sunpy.net.dataretriever.attrs.adapt
+   :headings: ^"
+
 JSOC
 ----
 

docs/topic_guide/extending_fido.rst

@@ -382,6 +382,7 @@ A simple example, which just checks the type of ``attrs`` and not their values w
         return supported_attrs.issuperset(query_attrs)
 
 Note, that this method is a class method, it gets called without instantiating your client to speed up the dispatching.
+If you are using the `~sunpy.net.dataretriever.client.GenericClient` as a base class, you do not need to implement this method, as it is already implemented in the base class.
 
 Writing a Fetch Method
 ----------------------

docs/tutorial/acquiring_data/index.rst

@@ -44,9 +44,10 @@ Fido supports a number of different remote data sources. To see a list the Fido
     For details of using `~sunpy.net.Fido` see :ref:`sunpy-tutorial-acquiring-data-index`.
     <BLANKLINE>
     <BLANKLINE>
-          Client                                                    Description
-    ----------------- -------------------------------------------------------------------------------------------------------
+          Client                                                            Description
+    ----------------- -----------------------------------------------------------------------------------------------------------------------
     CDAWEBClient      Provides access to query and download from the Coordinated Data Analysis Web (CDAWeb).
+    ADAPTClient       Provides access to the ADvanced Adaptive Prediction Technique (ADAPT) products of the National Solar Observatory (NSO).
     EVEClient         Provides access to Level 0CS Extreme ultraviolet Variability Experiment (EVE) data.
     GBMClient         Provides access to data from the Gamma-Ray Burst Monitor (GBM) instrument on board the Fermi satellite.
     XRSClient         Provides access to several GOES XRS files archive.
@@ -94,17 +95,17 @@ As an example:
     <BLANKLINE>
     Specifies the Instrument name for the search.
     <BLANKLINE>
-           Attribute Name          Client          Full Name                                           Description
-    --------------------------- ----------- ------------------------ --------------------------------------------------------------------------------
-    aia                         VSO         AIA                      Atmospheric Imaging Assembly
-    bcs                         VSO         BCS                      Bragg Crystal Spectrometer
-    be_continuum                VSO         BE-Continuum             INAF-OACT Barra Equatoriale Continuum Instrument
-    be_halpha                   VSO         BE-Halpha                INAF-OACT Barra Equatoriale Hα Instrument
-    bigbear                     VSO         Big Bear                 Big Bear Solar Observatory, California TON and GONG+ sites
-    caii                        VSO         CAII                     Kanzelhöhe Ca II k Instrument
-    cds                         VSO         CDS                      Coronal Diagnostic Spectrometer
-    celias                      VSO         CELIAS                   Charge, Element, and Isotope Analysis System
+           Attribute Name          Client   ...                                   Description
+    --------------------------- ----------- ... --------------------------------------------------------------------------------
+    adapt                       ADAPT       ... ADvanced Adaptive Prediction Technique.
+    aia                         VSO         ... Atmospheric Imaging Assembly
+    bcs                         VSO         ... Bragg Crystal Spectrometer
+    be_continuum                VSO         ... INAF-OACT Barra Equatoriale Continuum Instrument
+    be_halpha                   VSO         ... INAF-OACT Barra Equatoriale Hα Instrument
+    bigbear                     VSO         ... Big Bear Solar Observatory, California TON and GONG+ sites
     ...
+    xrs                         XRS         ... GOES X-ray Sensor
+    xrt                         VSO         ... X-Ray Telescope
 
 This is a full list of known values, a description, and which clients support those values (if you want to search using a specific data source).
 Printing attributes like this is supported for most attributes, including  client specific ones.

docs/whatsnew/6.0.rst

@@ -88,3 +88,9 @@ Deprecate positional arguments in :meth:`sunpy.map.GenericMap.plot`
 
 The arguments for :meth:`sunpy.map.GenericMap.plot` have been changed to being keyword only.
 Pass them as keyword arguments (e.g., ``..., title=True, ...``) instead.
+
+Support for ADvanced Adaptive Prediction Technique (ADAPT)
+==========================================================
+
+A new map source has been added (``sunpy.map.sources.ADAPTMap``) to support the ADAPT data files.
+In addition, a new client (`sunpy.net.dataretriever.ADAPTClient`) has been added to search and download ADAPT files.

sunpy/net/attrs.py

@@ -15,12 +15,12 @@
 In addition to the core attrs defined here, other sunpy clients also provide
 attrs specific to them, under:
 
-* `a.vso <sunpy.net.vso.attrs>`
-* `a.jsoc <sunpy.net.jsoc.attrs>`
+* `a.adapt <sunpy.net.dataretriever.attrs.adapt>`
 * `a.goes <sunpy.net.dataretriever.attrs.goes>`
 * `a.hek <sunpy.net.hek.attrs>`
 * `a.helio <sunpy.net.helio.attrs>`
-
+* `a.jsoc <sunpy.net.jsoc.attrs>`
+* `a.vso <sunpy.net.vso.attrs>`
 """
 from ._attrs import (
     Detector,

sunpy/net/dataretriever/__init__.py

@@ -9,6 +9,7 @@
 """
 
 from .client import *
+from .sources.adapt import *
 from .sources.eve import *
 from .sources.fermi_gbm import *
 from .sources.goes import *

sunpy/net/dataretriever/attrs/__init__.py

@@ -1,3 +1,3 @@
-from . import goes
+from . import adapt, goes
 
-__all__ = ['goes']
+__all__ = ['adapt', 'goes']

sunpy/net/dataretriever/attrs/adapt.py

@@ -0,0 +1,77 @@
+from sunpy.net.attr import SimpleAttr
+
+__all__ = ['ADAPTFileType', 'ADAPTLonType', 'ADAPTInputSource',
+           'ADAPTDataAssimilation', 'ADAPTResolution', 'ADAPTVersionYear', 'ADAPTVersionMonth',
+           'ADAPTEvolutionMode', 'ADAPTHelioData', 'ADAPTRealizations', 'ADAPTMagData']
+
+# Define a custom __dir__ to restrict tab-completion to __all__
+def __dir__():
+    return __all__
+
+
+class ADAPTFileType(SimpleAttr):
+    """
+    ADAPT file type: Public.
+    """
+    pass
+
+class ADAPTLonType(SimpleAttr):
+    """
+    ADAPT longitude type: Carrington Fixed, Central Meridian, East Limb.
+    """
+    pass
+
+class ADAPTInputSource(SimpleAttr):
+    """
+    ADAPT input source: All, KPVT, VSM, GONG, HMI, MDI, MWO.
+    """
+    pass
+
+class ADAPTDataAssimilation(SimpleAttr):
+    """
+    ADAPT data assimilation: WH, enLS, enkf, enLAKF.
+    """
+    pass
+
+class ADAPTResolution(SimpleAttr):
+    """
+    ADAPT model spatial resolution: 1.0 deg, 0.2 deg.
+    """
+    pass
+
+class ADAPTVersionYear(SimpleAttr):
+    """
+    ADAPT code version year.
+    """
+    pass
+
+class ADAPTVersionMonth(SimpleAttr):
+    """
+    ADAPT code version month.
+    """
+    pass
+
+class ADAPTEvolutionMode(SimpleAttr):
+    """
+    ADAPT evolution mode: Data assimilation step, Intermediate step, Forecast step.
+    """
+    pass
+
+class ADAPTHelioData(SimpleAttr):
+    """
+    ADAPT helioseismic data: Not added or no data, Far-side, Emergence, Both emergence & far-side.
+    """
+    pass
+
+class ADAPTRealizations(SimpleAttr):
+    """
+    ADAPT realizations: number of model/file realizations, e.g., 16 -> "016"
+    """
+    pass
+
+
+class ADAPTMagData(SimpleAttr):
+    """
+    ADAPT magnetic data: Not added or no data, Mag-los, Mag-vector, Mag- both los & vector, Mag- polar avg obs, Mag- los & polar, Mag- vector & polar, Mag- both los and vector & polar.
+    """
+    pass

sunpy/net/dataretriever/sources/adapt.py

@@ -0,0 +1,94 @@
+from sunpy.net import attrs as a
+from sunpy.net.dataretriever import GenericClient
+from sunpy.net.dataretriever.attrs.adapt import (
+    ADAPTDataAssimilation,
+    ADAPTEvolutionMode,
+    ADAPTFileType,
+    ADAPTHelioData,
+    ADAPTInputSource,
+    ADAPTLonType,
+    ADAPTMagData,
+    ADAPTRealizations,
+    ADAPTResolution,
+    ADAPTVersionMonth,
+    ADAPTVersionYear,
+)
+
+__all__ = ['ADAPTClient']
+
+
+class ADAPTClient(GenericClient):
+    """
+    Provides access to the ADvanced Adaptive Prediction Technique (ADAPT) products
+    of the National Solar Observatory (NSO).
+
+    `Searches data hosted by the NSO <https://gong.nso.edu/adapt/maps/gong/>`__
+
+    Examples
+    --------
+    >>> import astropy.units as u
+
+    >>> from sunpy.net import Fido, attrs as a
+    >>> from sunpy.coordinates.sun import carrington_rotation_time
+
+    >>> # Define the Carrington Rotation Number and the number of frames
+    >>> CR = 2193
+    >>> frames = 10
+    >>> date_start = carrington_rotation_time(CR)
+    >>> date_end = date_start + frames*(3*1.9999999 * u.hour)
+    >>> longitude_type = '0'
+
+    >>> Fido.search(a.Time(date_start, date_end), a.Instrument('adapt'), a.adapt.ADAPTLonType(longitude_type))  # doctest: +REMOTE_DATA
+    <sunpy.net.fido_factory.UnifiedResponse object at ...>
+    Results from 1 Provider:
+    <BLANKLINE>
+    10 Results from the ADAPTClient:
+    <BLANKLINE>
+           Start Time               End Time        Instrument Provider Source ... ADAPTMagData days_since_last_obs hours_since_last_obs minutes_since_last_obs seconds_since_last_obs
+    ----------------------- ----------------------- ---------- -------- ------ ... ------------ ------------------- -------------------- ---------------------- ----------------------
+    2017-07-20 08:00:00.000 2017-07-20 08:00:59.999      ADAPT      NSO   GONG ...            1                   0                    1                     56                      0
+    2017-07-20 14:00:00.000 2017-07-20 14:00:59.999      ADAPT      NSO   GONG ...            1                   0                    1                     56                      0
+    2017-07-20 20:00:00.000 2017-07-20 20:00:59.999      ADAPT      NSO   GONG ...            1                   0                    1                     56                      0
+    2017-07-21 02:00:00.000 2017-07-21 02:00:59.999      ADAPT      NSO   GONG ...            1                   0                    1                     56                      0
+    2017-07-21 08:00:00.000 2017-07-21 08:00:59.999      ADAPT      NSO   GONG ...            1                   0                    1                     56                      0
+    2017-07-21 14:00:00.000 2017-07-21 14:00:59.999      ADAPT      NSO   GONG ...            1                   0                    1                     56                      0
+    2017-07-21 20:00:00.000 2017-07-21 20:00:59.999      ADAPT      NSO   GONG ...            1                   0                    1                     56                      0
+    2017-07-22 02:00:00.000 2017-07-22 02:00:59.999      ADAPT      NSO   GONG ...            1                   0                    1                     56                      0
+    2017-07-22 08:00:00.000 2017-07-22 08:00:59.999      ADAPT      NSO   GONG ...            1                   0                    4                     36                      0
+    2017-07-22 14:00:00.000 2017-07-22 14:00:59.999      ADAPT      NSO   GONG ...            1                   0                    1                     56                      0
+    <BLANKLINE>
+    <BLANKLINE>
+
+    References
+    ----------
+    `Names and possible attrs values are available <https://gong.nso.edu/adapt/maps/adapt_filename_notes.txt>`__.
+    """
+    baseurl = r'https://gong.nso.edu/adapt/maps/gong/%Y/adapt(\d){5}_(\d){2}(\w){1}(\d){3}_(\d){12}_(\w){1}(\d){8}(\w){1}(\d){1}\.fts\.gz'
+    pattern = '{}adapt{ADAPTFileType:1d}{ADAPTLonType:1d}{ADAPTInputSource:1d}{ADAPTDataAssimilation:1d}{ADAPTResolution:1d}' + \
+    '_{ADAPTVersionYear:2d}{ADAPTVersionMonth:1l}{ADAPTRealizations:3d}_{year:4d}{month:2d}{day:2d}{hour:2d}{minute:2d}' + \
+    '_{ADAPTEvolutionMode:1l}{days_since_last_obs:2d}{hours_since_last_obs:2d}{minutes_since_last_obs:2d}{seconds_since_last_obs:2d}{ADAPTHelioData:1l}{ADAPTMagData:1d}.fts.gz'
+
+    @classmethod
+    def _attrs_module(cls):
+        return 'adapt', 'sunpy.net.dataretriever.attrs.adapt'
+
+    @classmethod
+    def register_values(cls):
+        adict = {
+            a.Instrument: [('ADAPT', 'ADvanced Adaptive Prediction Technique.')],
+            a.Provider: [('NSO', 'National Solar Observatory.')],
+            a.Source: [('GONG', 'Global Oscillation Network Group.')],
+            ADAPTFileType: [('4', 'Public')],
+            ADAPTLonType: [('0', 'Carrington Fixed'), ('1', 'Central Meridian'), ('2', 'East Limb')],
+            ADAPTInputSource: [('0', 'All'), ('1', 'KPVT'), ('2', 'VSM'), ('3', 'GONG'), ('4', 'HMI'), ('5', 'MDI'), ('6', 'MWO')],
+            ADAPTDataAssimilation: [('0', 'WH'), ('1', 'enLS'), ('2', 'enkf'), ('3', 'enLAKF')],
+            ADAPTResolution: [('1', '1.0 deg'), ('2', '0.2 deg')],
+            ADAPTVersionYear: [(str(i), f"Code version year -> {2000 + i}") for i in range(1, 20)],
+            ADAPTRealizations: [(str(i), f"Number of Realizations -> {i}") for i in range(1, 20)],
+            ADAPTVersionMonth: [(chr(i+96), f"Code version month -> {i}") for i in range(1, 13)],
+            ADAPTEvolutionMode: [('a', 'Data assimilation step'), ('i', 'Intermediate step'), ('f', 'Forecast step')],
+            ADAPTHelioData: [('n', 'Not added or no data'), ('f', 'Far-side'), ('e', 'Emergence'), ('b', 'Both emergence & far-side')],
+            ADAPTMagData: [('0', 'Not added or no data'), ('1', 'Mag-los'), ('2', 'Mag-vector'), ('3', 'Mag- both los & vector'),
+                            ('4', 'Mag- polar avg obs'), ('5', 'Mag- los & polar'), ('6', 'Mag- vector & polar'), ('7', 'Mag- both los and vector & polar')]
+        }
+        return adict

sunpy/net/dataretriever/sources/tests/test_adapt.py

@@ -0,0 +1,104 @@
+
+import tempfile
+
+import pytest
+from hypothesis import given
+
+import sunpy.net.dataretriever.sources.adapt as adapt
+from sunpy.net import attrs as a
+from sunpy.net.dataretriever.client import QueryResponse
+from sunpy.net.tests.strategies import time_attr
+from sunpy.time import parse_time
+
+
+@pytest.fixture
+def adapt_client():
+    return adapt.ADAPTClient()
+
+
+@given(time_attr())
+def test_can_handle_query(time):
+    # Hypothesis complains if we use the fixture
+    adapt_client = adapt.ADAPTClient()
+    ans1 = adapt_client._can_handle_query(time, a.Instrument.adapt)
+    assert ans1 is True
+    ans2 = adapt_client._can_handle_query(time, a.Instrument.adapt,
+                                         a.adapt.ADAPTResolution('1'))
+    assert ans2 is True
+    ans3 = adapt_client._can_handle_query(time, a.Instrument.adapt,
+                                         a.adapt.ADAPTResolution('1'),
+                                         a.adapt.ADAPTHelioData('f'))
+    assert ans3 is True
+    ans4 = adapt_client._can_handle_query(time)
+    assert ans4 is False
+    ans5 = adapt_client._can_handle_query(time, a.Instrument.adapt, a.Provider.nso)
+    assert ans5 is True
+    ans6 = adapt_client._can_handle_query(time, a.Instrument.adapt,
+                                         a.adapt.ADAPTLonType('0'))
+    assert ans6 is True
+
+
+def mock_query_object(adapt_client):
+    """
+    Creating a Query Response object and prefilling it with some information
+    """
+    start = '2019-05-25T02:00:00.00'
+    end = '2019-05-25T02:00:59.999'
+    obj = {
+        'Start Time': parse_time(start),
+        'End Time': parse_time(end),
+        'Instrument': 'ADAPT',
+        'Physobs': 'flux',
+        'Source': 'GONG',
+        'Provider': 'NSO',
+        'url': ("https://gong.nso.edu/adapt/maps/gong/2019/adapt40311_03i012_201905250200_i00005600n0.fts.gz")
+    }
+    results = QueryResponse([obj], client=adapt_client)
+    return results
+
+
+@pytest.mark.remote_data
+def test_fetch_working(adapt_client):
+    """
+    Tests if the online server is working.
+    This also checks if the mock is working well.
+    """
+    start = '2019/05/25 02:00:00'
+    end = '2019/05/26 02:00:59.999'
+    tr = a.Time(start, end)
+    qr = adapt_client.search(tr, a.Instrument.adapt)[0]
+    mock_qr = mock_query_object(adapt_client)[0]
+
+    assert mock_qr['Source'] == qr['Source']
+    assert mock_qr['Provider'] == qr['Provider']
+    assert mock_qr['Instrument'] == qr['Instrument']
+    assert mock_qr['url'] == qr['url']
+    assert qr['Start Time'].isot == mock_qr['Start Time'].isot
+    assert qr['End Time'].isot == mock_qr['End Time'].isot
+
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        download_list = adapt_client.fetch(qr, path=tmpdirname)
+    assert len(download_list) == 1
+
+
+
+def test_show(adapt_client):
+    mock_qr = mock_query_object(adapt_client)
+    qrshow0 = mock_qr.show()
+    qrshow1 = mock_qr.show('Start Time', 'Instrument')
+    allcols = {'Start Time', 'End Time', 'Instrument', 'Source', 'Provider', 'url'}
+    assert not allcols.difference(qrshow0.colnames)
+    assert qrshow1.colnames == ['Start Time', 'Instrument']
+    assert qrshow0['Instrument'][0] == 'ADAPT'
+
+
+def test_attr_reg():
+    assert a.Instrument.adapt == a.Instrument('ADAPT')
+
+
+def test_client_repr(adapt_client):
+    """
+    Repr check
+    """
+    output = str(adapt_client)
+    assert output[:50] == 'sunpy.net.dataretriever.sources.adapt.ADAPTClient\n'