dkist 1.9.1__tar.gz → 1.9.2__tar.gz

{dkist-1.9.1 → dkist-1.9.2}/CHANGELOG.rst

@@ -1,3 +1,12 @@
+1.9.2 (2025-01-29)
+==================
+
+Bug Fixes
+---------
+
+- Improve the ASDF detection code so out of date ASDF filenames generated by the DKIST data center are skipped if a newer filename is present. (`#503 <https://github.com/DKISTDC/dkist/pull/503>`__)
+
+
 1.9.1 (2025-01-15)
 ==================
 

{dkist-1.9.1/dkist.egg-info → dkist-1.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dkist
-Version: 1.9.1
+Version: 1.9.2
 Summary: DKIST User Tools
 Author-email: NSO / AURA <stuart@cadair.com>
 Project-URL: Homepage, https://github.com/DKISTDC/dkist

{dkist-1.9.1 → dkist-1.9.2}/dkist/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.9.1'
-__version_tuple__ = version_tuple = (1, 9, 1)
+__version__ = version = '1.9.2'
+__version_tuple__ = version_tuple = (1, 9, 2)

{dkist-1.9.1 → dkist-1.9.2}/dkist/data/api_search_values.json

@@ -1 +1 @@
-{"parameterValues":[{"parameterName":"createDateMin","values":{"minValue":"2022-12-08T19:07:55.038280","maxValue":"2025-01-15T00:09:37.896777"}},{"parameterName":"createDateMax","values":{"minValue":"2022-12-08T19:07:55.038280","maxValue":"2025-01-15T00:09:37.896777"}},{"parameterName":"endTimeMin","values":{"minValue":"2022-02-23T20:48:55.393500","maxValue":"2024-11-08T22:02:14.064417"}},{"parameterName":"endTimeMax","values":{"minValue":"2022-02-23T20:48:55.393500","maxValue":"2024-11-08T22:02:14.064417"}},{"parameterName":"exposureTimeMin","values":{"minValue":0.037,"maxValue":1380.2332394366197}},{"parameterName":"exposureTimeMax","values":{"minValue":0.037,"maxValue":1380.2332394366197}},{"parameterName":"instrumentNames","values":{"categoricalValues":["VBI","CRYO-NIRSP","VISP"]}},{"parameterName":"qualityAverageFriedParameterMin","values":{"minValue":0.01695005847191685,"maxValue":2.6520787500175156e+30}},{"parameterName":"qualityAverageFriedParameterMax","values":{"minValue":0.01695005847191685,"maxValue":2.6520787500175156e+30}},{"parameterName":"qualityAveragePolarimetricAccuracyMin","values":{"minValue":0.14787299538416715,"maxValue":263.313656131021}},{"parameterName":"qualityAveragePolarimetricAccuracyMax","values":{"minValue":0.14787299538416715,"maxValue":263.313656131021}},{"parameterName":"startTimeMin","values":{"minValue":"2022-02-23T19:05:32.338002","maxValue":"2024-11-08T21:40:00.522500"}},{"parameterName":"startTimeMax","values":{"minValue":"2022-02-23T19:05:32.338002","maxValue":"2024-11-08T21:40:00.522500"}},{"parameterName":"targetTypes","values":{"categoricalValues":["coronalhole","activecorona","quietcorona","prominence","sunspot","spicules","quietsun","plages","unknown","filament"]}},{"parameterName":"averageDatasetSpectralSamplingMin","values":{"minValue":0.000540156130946172,"maxValue":0.004605031614270178}},{"parameterName":"averageDatasetSpectralSamplingMax","values":{"minValue":0.000540156130946172,"maxValue":0.004605031614270178}},{"parameterName":"averageDatasetSpatialSamplingMin","values":{"minValue":0.0,"maxValue":12388.04306084}},{"parameterName":"averageDatasetSpatialSamplingMax","values":{"minValue":0.0,"maxValue":12388.04306084}},{"parameterName":"averageDatasetTemporalSamplingMin","values":{"minValue":2.66666599999823,"maxValue":5263.145059399398}},{"parameterName":"averageDatasetTemporalSamplingMax","values":{"minValue":2.66666599999823,"maxValue":5263.145059399398}},{"parameterName":"highLevelSoftwareVersion","values":{"categoricalValues":["Alakai_10-0","Pono_6.1.5","Pono_2.1.0","Alakai_5-1","Pono_3.1.0","Pono_8.0.0","Alakai_11.1.0","Alakai_6-0","Alakai_8-0","Alakai_7-0","Pono_7.0.0","Pono_1.0.0","Alakai_3-0","Alakai_4-0","Pono_9.1.0"]}},{"parameterName":"workflowName","values":{"categoricalValues":["l0_to_l1_visp","l0_to_l1_cryonirsp_sp","l0_to_l1_cryonirsp_ci","l0_to_l1_vbi_summit-calibrated"]}},{"parameterName":"workflowVersion","values":{"categoricalValues":["1.18.6","2.16.7","1.18.0","2.0.2","1.11.1","2.20.9","2.19.5","1.2.1","2.7.5","2.19.4","2.3.1","2.19.3","1.15.0","1.0.0","1.15.1","1.3.5","1.18.4","2.16.0","1.2.0","2.10.2","2.7.4","2.10.3","2.20.8","2.16.4","1.1.7","2.17.0","0.16.0","1.3.2","1.17.5","1.1.10","2.15.0","2.20.7","2.7.2","2.10.15","1.4.9","1.1.2","2.7.0","2.19.1","2.20.5","2.10.1","2.7.3","1.12.1","2.10.12","1.1.5","1.3.3","2.6.1","2.10.13","2.19.2","2.0.1","1.18.5","1.4.8","1.8.2","2.9.0","2.10.7","2.3.0","1.17.1","2.10.0","1.13.0","2.20.6","1.14.7","1.4.11","2.20.3","2.13.1","1.4.1"]}},{"parameterName":"headerDataUnitCreationDateMin","values":{"minValue":"2022-12-08T17:30:30.462000","maxValue":"2025-01-14T23:42:16.012000"}},{"parameterName":"headerDataUnitCreationDateMax","values":{"minValue":"2022-12-08T17:30:30.462000","maxValue":"2025-01-14T23:42:16.012000"}},{"parameterName":"headerVersion","values":{"categoricalValues":["4.3.0","4.6.0","4.0.0","4.1.1","3.4.0","3.9.0","4.7.0","4.10.0","3.5.0","4.2.0","3.6.0","3.3.0","3.0.0","3.7.1","3.8.1","4.5.0"]}},{"parameterName":"spectralLines","values":{"categoricalValues":["Na I D2 (589.0 nm)","Sr I (460.73 nm)","Ca II (854.21 nm)","Fe I (630.15 nm)","Ca II K (393.37 nm)","Ca II H (396.85 nm)","Na I D1 (589.59 nm)","Mg I b1 (517.27 nm)","Fe I (630.25 nm)","Fe XIII (1079.8 nm)","Fe XIII (1074.7 nm)","H epsilon (397.0 nm)","Fe I (525.04 nm)","H beta (486.13 nm)","He I (1083.0 nm)","H alpha (656.28 nm)"]}}]}
+{"parameterValues":[{"parameterName":"createDateMin","values":{"minValue":"2022-12-08T19:07:55.038280","maxValue":"2025-01-28T07:25:53.776348"}},{"parameterName":"createDateMax","values":{"minValue":"2022-12-08T19:07:55.038280","maxValue":"2025-01-28T07:25:53.776348"}},{"parameterName":"endTimeMin","values":{"minValue":"2022-02-23T20:48:55.393500","maxValue":"2024-11-08T22:02:14.064417"}},{"parameterName":"endTimeMax","values":{"minValue":"2022-02-23T20:48:55.393500","maxValue":"2024-11-08T22:02:14.064417"}},{"parameterName":"exposureTimeMin","values":{"minValue":0.037,"maxValue":1380.2332394366197}},{"parameterName":"exposureTimeMax","values":{"minValue":0.037,"maxValue":1380.2332394366197}},{"parameterName":"instrumentNames","values":{"categoricalValues":["VBI","CRYO-NIRSP","VISP"]}},{"parameterName":"qualityAverageFriedParameterMin","values":{"minValue":0.01695005847191685,"maxValue":2.6520787500175156e+30}},{"parameterName":"qualityAverageFriedParameterMax","values":{"minValue":0.01695005847191685,"maxValue":2.6520787500175156e+30}},{"parameterName":"qualityAveragePolarimetricAccuracyMin","values":{"minValue":0.14787299538416715,"maxValue":263.313656131021}},{"parameterName":"qualityAveragePolarimetricAccuracyMax","values":{"minValue":0.14787299538416715,"maxValue":263.313656131021}},{"parameterName":"startTimeMin","values":{"minValue":"2022-02-23T19:05:32.338002","maxValue":"2024-11-08T21:40:00.522500"}},{"parameterName":"startTimeMax","values":{"minValue":"2022-02-23T19:05:32.338002","maxValue":"2024-11-08T21:40:00.522500"}},{"parameterName":"targetTypes","values":{"categoricalValues":["coronalhole","activecorona","quietcorona","prominence","sunspot","spicules","quietsun","plages","unknown","filament"]}},{"parameterName":"averageDatasetSpectralSamplingMin","values":{"minValue":0.000540156130946172,"maxValue":0.004605031614270178}},{"parameterName":"averageDatasetSpectralSamplingMax","values":{"minValue":0.000540156130946172,"maxValue":0.004605031614270178}},{"parameterName":"averageDatasetSpatialSamplingMin","values":{"minValue":0.0,"maxValue":12388.04306084}},{"parameterName":"averageDatasetSpatialSamplingMax","values":{"minValue":0.0,"maxValue":12388.04306084}},{"parameterName":"averageDatasetTemporalSamplingMin","values":{"minValue":2.66666599999823,"maxValue":5263.145059399398}},{"parameterName":"averageDatasetTemporalSamplingMax","values":{"minValue":2.66666599999823,"maxValue":5263.145059399398}},{"parameterName":"highLevelSoftwareVersion","values":{"categoricalValues":["Alakai_10-0","Pono_6.1.5","Pono_2.1.0","Alakai_5-1","Pono_3.1.0","Pono_8.0.0","Alakai_11.1.0","Alakai_6-0","Alakai_8-0","Alakai_7-0","Pono_7.0.0","Pono_1.0.0","Alakai_3-0","Alakai_4-0","Pono_9.1.0"]}},{"parameterName":"workflowName","values":{"categoricalValues":["l0_to_l1_visp","l0_to_l1_cryonirsp_sp","l0_to_l1_cryonirsp_ci","l0_to_l1_vbi_summit-calibrated"]}},{"parameterName":"workflowVersion","values":{"categoricalValues":["1.18.6","2.16.7","1.18.0","2.0.2","1.11.1","2.20.9","2.19.5","1.2.1","2.7.5","2.19.4","2.3.1","2.19.3","1.15.0","1.0.0","1.15.1","1.3.5","1.18.4","2.16.0","1.2.0","2.10.2","2.7.4","1.18.8","2.10.3","2.20.8","2.16.4","1.1.7","2.17.0","0.16.0","1.3.2","1.17.5","1.1.10","2.15.0","2.20.7","2.7.2","2.10.15","1.4.9","1.1.2","2.7.0","2.19.1","2.20.5","2.20.10","2.10.1","2.7.3","1.12.1","2.10.12","1.1.5","1.3.3","2.6.1","2.10.13","2.19.2","2.0.1","1.18.5","1.4.8","1.8.2","2.9.0","2.10.7","2.3.0","2.10.0","1.17.1","1.13.0","2.20.6","1.14.7","1.4.11","2.20.3","2.13.1","1.4.1"]}},{"parameterName":"headerDataUnitCreationDateMin","values":{"minValue":"2022-12-08T17:30:30.462000","maxValue":"2025-01-28T00:07:38.944000"}},{"parameterName":"headerDataUnitCreationDateMax","values":{"minValue":"2022-12-08T17:30:30.462000","maxValue":"2025-01-28T00:07:38.944000"}},{"parameterName":"headerVersion","values":{"categoricalValues":["4.3.0","4.6.0","4.0.0","4.1.1","3.4.0","3.9.0","4.7.0","4.10.0","3.5.0","4.2.0","3.6.0","3.3.0","3.0.0","3.7.1","3.8.1","4.5.0"]}},{"parameterName":"spectralLines","values":{"categoricalValues":["Na I D2 (589.0 nm)","Sr I (460.73 nm)","Ca II (854.21 nm)","Fe I (630.15 nm)","Ca II K (393.37 nm)","Ca II H (396.85 nm)","Na I D1 (589.59 nm)","Mg I b1 (517.27 nm)","Fe I (630.25 nm)","Fe XIII (1079.8 nm)","Fe XIII (1074.7 nm)","H epsilon (397.0 nm)","Fe I (525.04 nm)","H beta (486.13 nm)","He I (1083.0 nm)","H alpha (656.28 nm)"]}}]}

{dkist-1.9.1 → dkist-1.9.2}/dkist/dataset/loader.py

@@ -1,11 +1,16 @@
+import re
+import warnings
 import importlib.resources as importlib_resources
 from pathlib import Path
 from functools import singledispatch
+from collections import defaultdict
 
 from parfive import Results
 
 import asdf
 
+from dkist.utils.exceptions import DKISTUserWarning
+
 try:
     # first try to import from asdf.exceptions for asdf 2.15+
     from asdf.exceptions import ValidationError
@@ -14,6 +19,9 @@ except ImportError:
     from asdf import ValidationError
 
 
+ASDF_FILENAME_PATTERN = r"^(?P<instrument>[A-Z-]+)_L1_(?P<timestamp>\d{8}T\d{6})_(?P<datasetid>[A-Z]{5,})(?P<suffix>_user_tools|_metadata)?.asdf$"
+
+
 def asdf_open_memory_mapping_kwarg(memmap: bool) -> dict:
     if asdf.__version__ > "3.1.0":
         return {"memmap": memmap}
@@ -138,8 +146,30 @@ def _load_from_path(path: Path):
 
 def _load_from_directory(directory):
     """
-    Construct a `~dkist.dataset.Dataset` from a directory containing one
-    asdf file and a collection of FITS files.
+    Construct a `~dkist.dataset.Dataset` from a directory containing one (or
+    more) ASDF files and a collection of FITS files.
+
+    ASDF files have the generic pattern:
+
+    ``{instrument}_L1_{start_time:%Y%m%dT%H%M%S}_{dataset_id}[_{suffix}].asdf``
+
+    where the ``_{suffix}`` on the end may be absent or one of a few different
+    suffixes which have been used at different times.  When searching a
+    directory for one or more ASDF file to load we should attempt to only load
+    one per dataset ID by selecting files in suffix order.
+
+    The order of suffixes are (from newest used to oldest):
+
+    - ``_metadata``
+    - ``_user_tools``
+    - None
+
+    The algorithm used to find ASDF files to load in a directory is therefore:
+
+    - Glob the directory for all ASDF files
+    - Group all results by the filename up to and including the dataset id in the filename
+    - Ignore any ASDF files with an old suffix if a new suffix is present
+    - Throw a warning to the user if any ASDF files with older suffixes are found
     """
     base_path = Path(directory).expanduser()
     asdf_files = tuple(base_path.glob("*.asdf"))
@@ -147,12 +177,60 @@ def _load_from_directory(directory):
     if not asdf_files:
         raise ValueError(f"No asdf file found in directory {base_path}.")
 
-    if len(asdf_files) > 1:
-        return _load_from_iterable(asdf_files)
-
-    asdf_file = asdf_files[0]
+    if len(asdf_files) == 1:
+        return _load_from_asdf(asdf_files[0])
+
+    pattern = re.compile(ASDF_FILENAME_PATTERN)
+    candidates = []
+    asdfs_to_load = []
+    for filepath in asdf_files:
+        filename = filepath.name
+
+        # If the asdf file doesn't match the data center pattern then we load it
+        # as it's probably a custom user file
+        if pattern.match(filename) is None:
+            asdfs_to_load.append(filepath)
+            continue
+
+        # All the matches have to be checked
+        candidates.append(filepath)
+
+    # If we only have one match load it
+    if len(candidates) == 1:
+        asdfs_to_load += candidates
+    else:
+        # Now we group by prefix
+        matches = [pattern.match(fp.name) for fp in candidates]
+        grouped = defaultdict(list)
+        for m in matches:
+            prefix = m.string.removesuffix(".asdf").removesuffix(m.group("suffix") or "")
+            grouped[prefix].append(m.group("suffix"))
+
+        # Now we select the best suffix for each prefix
+        for prefix, suffixes in grouped.items():
+            if "_metadata" in suffixes:
+                asdfs_to_load.append(base_path / f"{prefix}_metadata.asdf")
+            elif "_user_tools" in suffixes:
+                asdfs_to_load.append(base_path / f"{prefix}_user_tools.asdf")
+            elif None in suffixes:
+                asdfs_to_load.append(base_path / f"{prefix}.asdf")
+            else:
+                # This branch should never be hit because the regex enumerates the suffixes
+                raise ValueError("Unknown suffix encountered.")  # pragma: no cover
+
+    # Throw a warning if we have skipped any files
+    if ignored_files := set(asdf_files).difference(asdfs_to_load):
+        warnings.warn(
+            f"ASDF files with old names ({', '.join([a.name for a in ignored_files])}) "
+            "were found in this directory and ignored. You may want to delete these files.",
+            DKISTUserWarning
+        )
+
+    if len(asdfs_to_load) == 1:
+        return _load_from_asdf(asdfs_to_load[0])
+
+    return _load_from_iterable(asdfs_to_load)
 
-    return _load_from_asdf(asdf_file)
 
 
 def _load_from_asdf(filepath):

dkist-1.9.2/dkist/dataset/tests/test_load_dataset.py

@@ -0,0 +1,203 @@
+import re
+import shutil
+import numbers
+
+import pytest
+from parfive import Results
+
+import asdf
+
+from dkist import Dataset, TiledDataset, load_dataset
+from dkist.data.test import rootdir
+from dkist.dataset.loader import ASDF_FILENAME_PATTERN
+from dkist.utils.exceptions import DKISTUserWarning
+
+
+@pytest.fixture
+def single_asdf_in_folder(tmp_path, asdf_path):
+    shutil.copy(asdf_path, tmp_path)
+    return tmp_path
+
+
+@pytest.fixture
+def multiple_asdf_in_folder(tmp_path, asdf_path):
+    shutil.copy(asdf_path, tmp_path)
+    shutil.copy(asdf_path, tmp_path  / "second_asdf.asdf")
+    return tmp_path
+
+
+@pytest.fixture
+def asdf_path():
+    return rootdir / "eit_dataset-1.1.0.asdf"
+
+
+@pytest.fixture
+def asdf_str(asdf_path):
+    return asdf_path.as_posix()
+
+
+@pytest.fixture
+def asdf_tileddataset_path():
+    return rootdir / "test_tiled_dataset-1.0.0_dataset-1.0.0.asdf"
+
+
+@pytest.fixture
+def single_asdf_in_folder_str(single_asdf_in_folder):
+    return single_asdf_in_folder.as_posix()
+
+
+@pytest.fixture
+def fixture_finder(request):
+    if isinstance(request.param, (list, tuple)):
+        return [request.getfixturevalue(i) for i in request.param]
+    return request.getfixturevalue(request.param)
+
+
+@pytest.mark.parametrize("fixture_finder", [
+        "asdf_path",
+        "asdf_str",
+        "single_asdf_in_folder",
+        "single_asdf_in_folder_str",
+    ],
+    indirect=True
+)
+def test_load_single_dataset(fixture_finder):
+    ds = load_dataset(fixture_finder)
+    assert isinstance(ds, Dataset)
+
+
+@pytest.mark.parametrize("fixture_finder", [
+        ["asdf_path", "asdf_str", "single_asdf_in_folder", "single_asdf_in_folder_str"],
+        ("asdf_path", "asdf_str", "single_asdf_in_folder", "single_asdf_in_folder_str"),
+    ],
+    indirect=True
+)
+def test_load_multiple(fixture_finder):
+    datasets = load_dataset(fixture_finder)
+    assert isinstance(datasets, list)
+    assert all(isinstance(ds, Dataset) for ds in datasets)
+
+
+def test_load_from_results(asdf_path, asdf_str):
+    res = Results([asdf_path])
+    ds = load_dataset(res)
+    assert isinstance(ds, Dataset)
+
+    res = Results([asdf_str, asdf_str])
+    ds = load_dataset(res)
+    assert isinstance(ds, list)
+    assert all(isinstance(ds, Dataset) for ds in ds)
+
+
+def test_multiple_from_dir(multiple_asdf_in_folder):
+    ds = load_dataset(multiple_asdf_in_folder)
+    assert isinstance(ds, list)
+    assert len(ds) == 2
+    assert all(isinstance(d, Dataset) for d in ds)
+
+
+def test_tiled_dataset(asdf_tileddataset_path):
+    ds = load_dataset(asdf_tileddataset_path)
+    assert isinstance(ds, TiledDataset)
+
+
+def test_errors(tmp_path):
+    with pytest.raises(TypeError, match="Input type dict"):
+        load_dataset({})
+
+    with pytest.raises(ValueError, match="No asdf file found"):
+        load_dataset(tmp_path)
+
+    with pytest.raises(ValueError, match="does not exist"):
+        load_dataset(tmp_path / "asdf")
+
+
+def test_not_dkist_asdf(tmp_path):
+    af = asdf.AsdfFile({"hello": "world"})
+    af.write_to(tmp_path / "test.asdf")
+
+    with pytest.raises(TypeError, match="not a valid DKIST"):
+        load_dataset(tmp_path / "test.asdf")
+
+
+def generate_asdf_folder(tmp_path, asdf_path, filenames):
+    for fname in filenames:
+        shutil.copy(asdf_path, tmp_path / fname)
+
+    return tmp_path
+
+
+@pytest.mark.parametrize(("filename", "match"), [
+    ("VBI_L1_20231016T184519_AJQWW.asdf", True),
+    ("VBI_L1_20231016T184519_AAAA.asdf", False),
+    ("VBI_L1_20231016T184519_AJQWW_user_tools.asdf", True),
+    ("VBI_L1_20231016T184519_AJQWW_metadata.asdf", True),
+    ("DL-NIRSP_L1_20231016T184519_AJQWW.asdf", True),
+    ("DL-NIRSP_L1_20231016T184519_AJQWW_user_tools.asdf", True),
+    ("DL-NIRSP_L1_20231016T184519_AJQWW_metadata.asdf", True),
+    ("VISP_L1_99999999T184519_AAAAAAA.asdf", True),
+    ("VISP_L1_20231016T888888_AAAAAAA_user_tools.asdf", True),
+    ("VISP_L1_20231016T184519_AAAAAAA_metadata.asdf", True),
+    ("VISP_L1_20231016T184519_AAAAAAA_unknown.asdf", False),
+    ("VISP_L1_20231016T184519.asdf", False),
+    ("wibble.asdf", False),
+    ])
+def test_asdf_regex(filename, match):
+    m = re.match(ASDF_FILENAME_PATTERN, filename)
+    assert bool(m) is match
+
+
+@pytest.mark.parametrize(("filenames", "indices"), [
+    pytest.param(("VBI_L1_20231016T184519_AJQWW.asdf",), 0, id="Single no suffix"),
+    pytest.param(("VBI_L1_20231016T184519_AJQWW_user_tools.asdf",), 0, id="single _user_tools"),
+    pytest.param(("VBI_L1_20231016T184519_AJQWW_metadata.asdf",), 0, id="single _metadata"),
+    pytest.param(("VBI_L1_20231016T184519_AJQWW_unknown.asdf",), 0, id="single _unknown"),
+    pytest.param(("VBI_L1_20231016T184519_AJQWW.asdf",
+                  "VBI_L1_20231016T184519_AJQWW_user_tools.asdf",), 1, id="none & _user_tools"),
+    pytest.param(("VBI_L1_20231016T184519_AJQWW.asdf",
+                  "VBI_L1_20231016T184519_AJQWW_user_tools.asdf",
+                  "VBI_L1_20231016T184519_AJQWW_metadata.asdf",), 2, id="_user_tools & _metadata"),
+    pytest.param(("VBI_L1_20231016T184519_AJQWW.asdf",
+                  "VBI_L1_20231016T184519_AJQWW_user_tools.asdf",
+                  "VBI_L1_20231016T184519_AJQWW_metadata.asdf",
+                  "VBI_L1_20231016T184519_AJQWW_unknown.asdf"), (2, 3), id="_user_tools & _metadata & _unknown"),
+    pytest.param(("random.asdf",
+                  "VBI_L1_20231016T184519_AJQWW_user_tools.asdf",), (0, 1), id="other pattern & _user_tools"),
+    pytest.param(("random.asdf",
+                  "VBI_L1_not_a_proper_name.asdf",
+                  "VBI_L1_20231016T184519_AJQWW_user_tools.asdf",
+                  "VBI_L1_20231016T184519_AJQWW_metadata.asdf",), (0, 1, 3), id="2 other patterns & _user_tools & _metadata"),
+    pytest.param(("VBI_L1_20231016T184519_AJQWW.asdf",
+                  "VISP_L1_20231016T184519_AJQWW.asdf",), (0, 1), id="Two patterns, no suffix"),
+    pytest.param(("VBI_L1_20231016T184519_AAAAA.asdf",
+                  "VBI_L1_20231016T184519_AAAAA_metadata.asdf",
+                  "VBI_L1_20231116T184519_BBBBBBB.asdf",
+                  "VBI_L1_20231216T184519_CCCCCCC.asdf",
+                  "VBI_L1_20231216T184519_CCCCCCC_user_tools.asdf"), (1, 2, 4), id="Three patterns, mixed suffixes"),
+])
+def test_select_asdf(tmp_path, asdf_path, filenames, indices, mocker):
+    asdf_folder = generate_asdf_folder(tmp_path, asdf_path, filenames)
+
+    asdf_file_paths = tuple(asdf_folder / fname for fname in filenames)
+
+    load_from_asdf = mocker.patch("dkist.dataset.loader._load_from_asdf")
+    load_from_iterable = mocker.patch("dkist.dataset.loader._load_from_iterable")
+
+    # The load_dataset call should throw a warning if any files are skipped, but
+    # not otherwise, the warning should have the filenames of any skipped files in
+    tuple_of_indices = indices if isinstance(indices, tuple) else (indices,)
+    if len(tuple_of_indices) == len(filenames):
+        datasets = load_dataset(asdf_folder)
+    else:
+        files_to_be_skipped = set(filenames).difference([filenames[i] for i in tuple_of_indices])
+        with pytest.warns(DKISTUserWarning, match=f".*[{'|'.join([re.escape(f) for f in files_to_be_skipped])}].*"):
+            datasets = load_dataset(asdf_folder)
+
+    if isinstance(indices, numbers.Integral):
+        load_from_asdf.assert_called_once_with(asdf_file_paths[indices])
+    else:
+        calls = load_from_iterable.mock_calls
+        # We need to assert that _load_from_iterable is called with the right
+        # paths but in a order-invariant way.
+        assert len(calls) == 1
+        assert set(calls[0].args[0]) == {asdf_file_paths[i] for i in indices}

{dkist-1.9.1 → dkist-1.9.2}/dkist/dataset/tests/test_tiled_dataset.py

@@ -77,12 +77,16 @@ def test_tiled_dataset_from_components(dataset):
 @pytest.mark.parametrize("share_zscale", [True, False], ids=["share_zscale", "indpendent_zscale"])
 def test_tileddataset_plot(share_zscale):
     from dkist.data.sample import VBI_AJQWW
-    ds = load_dataset(VBI_AJQWW)
+    ori_ds = load_dataset(VBI_AJQWW)
     newtiles = []
-    for tile in ds.flat:
+    for tile in ori_ds.flat:
         newtiles.append(tile.rebin((1, 8, 8), operation=np.sum))
-    ds = TiledDataset(np.array(newtiles).reshape(ds.shape), inventory=newtiles[0].inventory)
-    fig = plt.figure(figsize=(600, 800))
+    # ndcube 2.3.0 introduced a deepcopy for rebin, this broke our dataset validation
+    # https://github.com/sunpy/ndcube/issues/815
+    for tile in newtiles:
+        tile.meta["inventory"] = ori_ds.inventory
+    ds = TiledDataset(np.array(newtiles).reshape(ori_ds.shape), inventory=newtiles[0].inventory)
+    fig = plt.figure(figsize=(12, 15))
     ds.plot(0, share_zscale=share_zscale)
     return plt.gcf()
 

{dkist-1.9.1 → dkist-1.9.2/dkist.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: dkist
-Version: 1.9.1
+Version: 1.9.2
 Summary: DKIST User Tools
 Author-email: NSO / AURA <stuart@cadair.com>
 Project-URL: Homepage, https://github.com/DKISTDC/dkist

{dkist-1.9.1 → dkist-1.9.2}/tox.ini

@@ -7,6 +7,7 @@ requires =
 envlist =
     py{310,311,312}
     py312-{devdeps,benchmarks,figure}
+    py312-figure-devdeps
     py310-oldestdeps
     codestyle
     build_docs{,-notebooks}

dkist-1.9.1/dkist/dataset/tests/test_load_dataset.py

@@ -1,116 +0,0 @@
-import shutil
-
-import pytest
-from parfive import Results
-
-import asdf
-
-from dkist import Dataset, TiledDataset, load_dataset
-from dkist.data.test import rootdir
-
-
-@pytest.fixture
-def single_asdf_in_folder(tmp_path, asdf_path):
-    shutil.copy(asdf_path, tmp_path)
-    return tmp_path
-
-
-@pytest.fixture
-def multiple_asdf_in_folder(tmp_path, asdf_path):
-    shutil.copy(asdf_path, tmp_path)
-    shutil.copy(asdf_path, tmp_path  / "second_asdf.asdf")
-    return tmp_path
-
-
-@pytest.fixture
-def asdf_path():
-    return rootdir / "eit_dataset-1.1.0.asdf"
-
-
-@pytest.fixture
-def asdf_str(asdf_path):
-    return asdf_path.as_posix()
-
-
-@pytest.fixture
-def asdf_tileddataset_path():
-    return rootdir / "test_tiled_dataset-1.0.0_dataset-1.0.0.asdf"
-
-
-@pytest.fixture
-def single_asdf_in_folder_str(single_asdf_in_folder):
-    return single_asdf_in_folder.as_posix()
-
-
-@pytest.fixture
-def fixture_finder(request):
-    if isinstance(request.param, (list, tuple)):
-        return [request.getfixturevalue(i) for i in request.param]
-    return request.getfixturevalue(request.param)
-
-
-@pytest.mark.parametrize("fixture_finder", [
-        "asdf_path",
-        "asdf_str",
-        "single_asdf_in_folder",
-        "single_asdf_in_folder_str",
-    ],
-    indirect=True
-)
-def test_load_single_dataset(fixture_finder):
-    ds = load_dataset(fixture_finder)
-    assert isinstance(ds, Dataset)
-
-
-@pytest.mark.parametrize("fixture_finder", [
-        ["asdf_path", "asdf_str", "single_asdf_in_folder", "single_asdf_in_folder_str"],
-        ("asdf_path", "asdf_str", "single_asdf_in_folder", "single_asdf_in_folder_str"),
-    ],
-    indirect=True
-)
-def test_load_multiple(fixture_finder):
-    datasets = load_dataset(fixture_finder)
-    assert isinstance(datasets, list)
-    assert all(isinstance(ds, Dataset) for ds in datasets)
-
-
-def test_load_from_results(asdf_path, asdf_str):
-    res = Results([asdf_path])
-    ds = load_dataset(res)
-    assert isinstance(ds, Dataset)
-
-    res = Results([asdf_str, asdf_str])
-    ds = load_dataset(res)
-    assert isinstance(ds, list)
-    assert all(isinstance(ds, Dataset) for ds in ds)
-
-
-def test_multiple_from_dir(multiple_asdf_in_folder):
-    ds = load_dataset(multiple_asdf_in_folder)
-    assert isinstance(ds, list)
-    assert len(ds) == 2
-    assert all(isinstance(d, Dataset) for d in ds)
-
-
-def test_tiled_dataset(asdf_tileddataset_path):
-    ds = load_dataset(asdf_tileddataset_path)
-    assert isinstance(ds, TiledDataset)
-
-
-def test_errors(tmp_path):
-    with pytest.raises(TypeError, match="Input type dict"):
-        load_dataset({})
-
-    with pytest.raises(ValueError, match="No asdf file found"):
-        load_dataset(tmp_path)
-
-    with pytest.raises(ValueError, match="does not exist"):
-        load_dataset(tmp_path / "asdf")
-
-
-def test_not_dkist_asdf(tmp_path):
-    af = asdf.AsdfFile({"hello": "world"})
-    af.write_to(tmp_path / "test.asdf")
-
-    with pytest.raises(TypeError, match="not a valid DKIST"):
-        load_dataset(tmp_path / "test.asdf")