dkist 1.9.0__tar.gz → 1.9.2__tar.gz

{dkist-1.9.0 → dkist-1.9.2}/.cruft.json

@@ -1,6 +1,6 @@
 {
   "template": "https://github.com/sunpy/package-template",
-  "commit": "aec53b81aed2e7e534045e59303d82712fe82fb1",
+  "commit": "d6b86e07efa31959f43e99ba32d323cf76736a05",
   "checkout": null,
   "context": {
     "cookiecutter": {
@@ -10,13 +10,20 @@
       "author_name": "NSO / AURA",
       "author_email": "stuart@cadair.com",
       "project_url": "https://github.com/DKISTDC/dkist",
+      "github_repo": "DKISTDC/dkist",
+      "sourcecode_url": "https://github.com/DKISTDC/dkist",
+      "download_url": "https://pypi.org/project/dkist",
+      "documentation_url": "https://docs.dkist.nso.edu/projects/python-tools/en/stable/",
+      "changelog_url": "https://docs.dkist.nso.edu/projects/python-tools/en/stable/whatsnew/changelog.html",
+      "issue_tracker_url": "https://github.com/DKISTDC/dkist/issues",
       "license": "BSD 3-Clause",
       "minimum_python_version": "3.10",
       "use_compiled_extensions": "n",
       "enable_dynamic_dev_versions": "y",
       "include_example_code": "n",
       "include_cruft_update_github_workflow": "y",
-      "_sphinx_theme": "alabaster",
+      "use_extended_ruff_linting": "y",
+      "_sphinx_theme": "sunpy",
       "_parent_project": "",
       "_install_requires": "",
       "_copy_without_render": [
@@ -24,7 +31,8 @@
         "docs/_static",
         ".github/workflows/sub_package_update.yml"
       ],
-      "_template": "https://github.com/sunpy/package-template"
+      "_template": "https://github.com/sunpy/package-template",
+      "_commit": "d6b86e07efa31959f43e99ba32d323cf76736a05"
     }
   },
   "directory": null

dkist-1.9.2/.github/workflows/build_workshop.yml

@@ -0,0 +1,50 @@
+name: Build Workshop Repo
+
+on:
+  # Allow manual runs through the web UI
+  workflow_dispatch:
+  # Push to the main branch of the workshop repo on release
+  release:
+    types: [published]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build_workshop:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Configure Git
+        run: |
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --global user.name "${{ github.actor }}"
+
+      - name: Install pipx
+        run: pip install -U pipx
+
+      - name: Clone workshop repository
+        uses: actions/checkout@v4
+        with:
+          repository: 'DKISTDC/DKIST-Workshop'
+          fetch-depth: 0
+          path: 'workshop'
+
+      - name: Run pipx
+        run: pipx run ./tools/build_workshop_materials.py docs/tutorial workshop
+
+      - name: Commit and Push
+        run: |
+          cd workshop
+          git add .
+          git commit -m "Update notebooks from user tools repo"
+          git push

{dkist-1.9.0 → dkist-1.9.2}/.github/workflows/prepare-release.yml

@@ -125,7 +125,7 @@ jobs:
 
             A draft release has been made with the changelog, release it after merge to trigger the release.
           labels: |
-            no changelog
+            No Changelog Entry Needed
 
       - name: Create draft GitHub Release
         uses: actions/github-script@v7

{dkist-1.9.0 → dkist-1.9.2}/.github/workflows/sub_package_update.yml

@@ -21,14 +21,6 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: true
-      matrix:
-        include:
-          - add-paths: .
-            body: apply the changes to this repo.
-            branch: cruft/update
-            commit-message: "Automatic package template update"
-            title: Updates from the package template
-
     steps:
       - uses: actions/checkout@v4
 
@@ -55,25 +47,49 @@ jobs:
           echo "has_changes=$CHANGES" >> "$GITHUB_OUTPUT"
 
       - name: Run update if available
+        id: cruft_update
         if: steps.check.outputs.has_changes == '1'
         run: |
           git config --global user.email "${{ github.actor }}@users.noreply.github.com"
           git config --global user.name "${{ github.actor }}"
 
-          cruft update --skip-apply-ask --refresh-private-variables
+          cruft_output=$(cruft update --skip-apply-ask --refresh-private-variables)
+          echo $cruft_output
           git restore --staged .
 
-      - name: Create pull request
+          if [[ "$cruft_output" == *"Failed to cleanly apply the update, there may be merge conflicts."* ]]; then
+            echo merge_conflicts=1 >> $GITHUB_OUTPUT
+          else
+            echo merge_conflicts=0 >> $GITHUB_OUTPUT
+          fi
+
+      - name: Check if only .cruft.json is modified
+        id: cruft_json
         if: steps.check.outputs.has_changes == '1'
+        run: |
+          git status --porcelain=1
+          if [[ "$(git status --porcelain=1)" == " M .cruft.json" ]]; then
+            echo "Only .cruft.json is modified. Exiting workflow early."
+            echo "has_changes=0" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_changes=1" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create pull request
+        if: steps.cruft_json.outputs.has_changes == '1'
         uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          add-paths: ${{ matrix.add-paths }}
-          commit-message: ${{ matrix.commit-message }}
-          branch: ${{ matrix.branch }}
+          add-paths: "."
+          commit-message: "Automatic package template update"
+          branch: "cruft/update"
           delete-branch: true
-          branch-suffix: timestamp
-          title: ${{ matrix.title }}
+          draft: ${{ steps.cruft_update.outputs.merge_conflicts == '1' }}
+          title: "Updates from the package template"
+          labels: |
+            No Changelog Entry Needed
           body: |
-            This is an autogenerated PR, which will ${{ matrix.body }}.
-            [Cruft](https://cruft.github.io/cruft/) has detected updates from the Package Template
+            This is an autogenerated PR, which will applies the latest changes from the [SunPy Package Template](https://github.com/sunpy/package-template).
+            If this pull request has been opened as a draft there are conflicts which need fixing.
+
+            **To run the CI on this pull request you will need to close it and reopen it.**

{dkist-1.9.0 → dkist-1.9.2}/.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
     # This should be before any formatting hooks like isort
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.6.9"
+    rev: "v0.8.6"
     hooks:
       - id: ruff
         args: ["--fix"]

dkist-1.9.2/.ruff.toml

@@ -0,0 +1,122 @@
+target-version = "py310"
+line-length = 120
+exclude = [
+  ".git,",
+  "__pycache__",
+  "build",
+  "dkist/version.py",
+  "dkist/_dev/",
+]
+
+[lint]
+select = [
+  "E",
+  "F",
+  "W",
+  "UP",
+  "PT",
+  "BLE",
+  # "A",
+  "C4",
+  "ICN",
+  "G",
+  "INP",
+  "Q",
+  "RSE",
+  "PIE",
+  "T20",
+  "RET",
+  "TID",
+  "PTH",
+  "PD",
+  "PLC",
+  "PLE",
+  "FLY",
+  "NPY",
+  # "PERF",
+  "RUF",
+]
+
+extend-ignore = [
+  # pycodestyle (E, W)
+  "E501", # ignore line length will use a formatter instead
+  "E702",
+  "E711",
+  "E741",
+  "F405",
+  "F403",
+  "F821",
+  "F841",
+  # pyupgrade (UP)
+  "UP038", # Use | in isinstance - not compatible with models and is slower
+  # pytest (PT)
+  "PT001", # Always use pytest.fixture()
+  "PT023", # Always use () on pytest decorators
+  # Returns (RET)
+  "RET502",  # Do not implicitly return None in function able to return non-None value
+  "RET503",  # Missing explicit return at the end of function able to return non-None value
+  # flake8-pie (PIE)
+  "PIE808", # Disallow passing 0 as the first argument to range
+  # flake8-use-pathlib (PTH)
+  "PTH123", # open() should be replaced by Path.open()
+  # Ruff (RUF)
+  "RUF003", # Ignore ambiguous quote marks, doesn't allow ' in comments
+  "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar`
+  "RUF013", # PEP 484 prohibits implicit `Optional`
+  "RUF015", # Prefer `next(iter(...))` over single element slice
+]
+
+[lint.per-file-ignores]
+"setup.py" = [
+  "INP001", # File is part of an implicit namespace package.
+]
+"conftest.py" = [
+  "INP001", # File is part of an implicit namespace package.
+]
+"docs/conf.py" = [
+  "E402", # Module imports not at top of file
+  "A001",
+  "BLE001",
+  "T201",
+]
+"docs/*.py" = [
+  "INP001", # File is part of an implicit namespace package.
+]
+"examples/**.py" = [
+  "T201", # allow use of print in examples
+  "INP001", # File is part of an implicit namespace package.
+]
+"__init__.py" = [
+  "E402", # Module level import not at top of cell
+  "F401", # Unused import
+  "F403", # from {name} import * used; unable to detect undefined names
+  "F405", # {name} may be undefined, or defined from star imports
+]
+"test_*.py" = [
+  "E402", # Module level import not at top of cell
+  "B011",
+  "D",
+  "E402",
+  "PGH001",
+  "S101",
+]
+"dkist/logger.py" = [
+  "PTH",
+  "BLE001",
+]
+"dkist/tests/generate_*" = [
+  "T20",
+]
+"dkist/utils/sysinfo.py" = [
+  "T20",
+]
+"dkist/net/globus/auth.py" = [
+  "T201",
+]
+
+[lint.flake8-import-conventions.extend-aliases]
+"astropy.units" = "u"
+"sunpy.net.attrs" = "a"
+
+[lint.pydocstyle]
+convention = "numpy"

{dkist-1.9.0 → dkist-1.9.2}/CHANGELOG.rst

@@ -1,3 +1,35 @@
+1.9.2 (2025-01-29)
+==================
+
+Bug Fixes
+---------
+
+- Improve the ASDF detection code so out of date ASDF filenames generated by the DKIST data center are skipped if a newer filename is present. (`#503 <https://github.com/DKISTDC/dkist/pull/503>`__)
+
+
+1.9.1 (2025-01-15)
+==================
+
+Bug Fixes
+---------
+
+- Fix some small issues with `Dataset.__repr__`. (`#479 <https://github.com/DKISTDC/dkist/pull/479>`__)
+- Fix path errors with Globus transfers on windows. (`#489 <https://github.com/DKISTDC/dkist/pull/489>`__)
+
+
+Trivial/Internal Changes
+------------------------
+
+- Add tests for cropping datasets by world coords (`#310 <https://github.com/DKISTDC/dkist/pull/310>`__)
+- Fix small bug which caused `ds.flat` to break if not indexed. (`#475 <https://github.com/DKISTDC/dkist/pull/475>`__)
+
+
+Performance Improvements
+------------------------
+
+- Improve the performance of the ``TiledDataset`` ``repr`` and ``str``. (`#467 <https://github.com/DKISTDC/dkist/pull/467>`__)
+
+
 1.9.0 (2024-11-04)
 ==================
 

{dkist-1.9.0/dkist.egg-info → dkist-1.9.2}/PKG-INFO

@@ -1,9 +1,14 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dkist
-Version: 1.9.0
+Version: 1.9.2
 Summary: DKIST User Tools
 Author-email: NSO / AURA <stuart@cadair.com>
-Project-URL: repository, https://github.com/DKISTDC/dkist
+Project-URL: Homepage, https://github.com/DKISTDC/dkist
+Project-URL: Source Code, https://github.com/DKISTDC/dkist
+Project-URL: Download, https://pypi.org/project/dkist
+Project-URL: Documentation, https://docs.dkist.nso.edu/projects/python-tools/en/stable/
+Project-URL: Changelog, https://docs.dkist.nso.edu/projects/python-tools/en/stable/whatsnew/changelog.html
+Project-URL: Issue Tracker, https://github.com/DKISTDC/dkist/issues
 Requires-Python: >=3.10
 Description-Content-Type: text/x-rst
 License-File: LICENSE.rst
@@ -34,7 +39,7 @@ Requires-Dist: pytest-mock; extra == "tests"
 Requires-Dist: pytest-mpl; extra == "tests"
 Requires-Dist: pytest-httpserver; extra == "tests"
 Requires-Dist: pytest-filter-subpackage; extra == "tests"
-Requires-Dist: pytest-benchmark<5; extra == "tests"
+Requires-Dist: pytest-benchmark; extra == "tests"
 Requires-Dist: pytest-xdist; extra == "tests"
 Requires-Dist: hypothesis; extra == "tests"
 Requires-Dist: tox; extra == "tests"
@@ -42,10 +47,11 @@ Requires-Dist: pydot; extra == "tests"
 Provides-Extra: docs
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-automodapi; extra == "docs"
+Requires-Dist: sphinx-changelog; extra == "docs"
+Requires-Dist: sunpy-sphinx-theme; extra == "docs"
 Requires-Dist: packaging; extra == "docs"
 Requires-Dist: sphinx-astropy; extra == "docs"
 Requires-Dist: sphinx-gallery; extra == "docs"
-Requires-Dist: sphinx-changelog; extra == "docs"
 Requires-Dist: pytest; extra == "docs"
 Requires-Dist: sphinx_autodoc_typehints; extra == "docs"
 Requires-Dist: dkist-sphinx-theme>=1.1.2; extra == "docs"

{dkist-1.9.0 → dkist-1.9.2}/changelog/README.rst

@@ -21,6 +21,7 @@ Each file should be named like ``<PULL REQUEST>.<TYPE>.rst``, where ``<PULL REQU
 * ``breaking``: A change which requires users to change code and is not backwards compatible. (Not to be used for removal of deprecated features.)
 * ``feature``: New user facing features and any new behavior.
 * ``bugfix``: Fixes a reported bug.
+* ``performance``: A performance improvement which does not change behaviour.
 * ``doc``: Documentation addition or improvement, like rewording an entire session or adding missing docs.
 * ``removal``: Feature deprecation and/or feature removal.
 * ``trivial``: A change which has no user facing effect or is tiny change.

{dkist-1.9.0 → dkist-1.9.2}/dkist/__init__.py

@@ -17,7 +17,7 @@ except PackageNotFoundError:
     __version__ = "unknown"
 
 
-__all__ = ["TiledDataset", "Dataset", "load_dataset", "system_info", "conf"]
+__all__ = ["Dataset", "TiledDataset", "conf", "load_dataset", "system_info"]
 
 
 def write_default_config(overwrite=False):

{dkist-1.9.0 → dkist-1.9.2}/dkist/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.9.0'
-__version_tuple__ = version_tuple = (1, 9, 0)
+__version__ = version = '1.9.2'
+__version_tuple__ = version_tuple = (1, 9, 2)

{dkist-1.9.0 → dkist-1.9.2}/dkist/conftest.py

@@ -407,3 +407,21 @@ def pytest_runtest_call(item):
         # TODO raise a warning here
         warnings.warn("--ds and --tiled-ds were not found. Any supplied datasets will not be used.")
         yield item
+
+
+@pytest.fixture(scope="session")
+def croppable_visp_dataset(tmp_path_factory):
+    vispdir = tmp_path_factory.mktemp("data")
+    # This asdf file is for dataset ID BKEWK
+    with gzip.open(Path(rootdir) / "croppable_visp.asdf.gz", mode="rb") as gfo:
+        with open(vispdir / "croppable_visp.asdf", mode="wb") as afo:
+            afo.write(gfo.read())
+    return load_dataset(vispdir / "croppable_visp.asdf")
+
+
+@pytest.fixture(scope="session")
+def croppable_cryo_dataset():
+    with gzip.open(Path(rootdir) / "croppable_cryo.asdf.gz", mode="rb") as gfo:
+        with open(rootdir / "croppable_cryo.asdf", mode="wb") as afo:
+            afo.write(gfo.read())
+    return load_dataset(Path(rootdir) / "croppable_cryo.asdf")

dkist-1.9.2/dkist/data/api_search_values.json

@@ -0,0 +1 @@
+{"parameterValues":[{"parameterName":"createDateMin","values":{"minValue":"2022-12-08T19:07:55.038280","maxValue":"2025-01-28T07:25:53.776348"}},{"parameterName":"createDateMax","values":{"minValue":"2022-12-08T19:07:55.038280","maxValue":"2025-01-28T07:25:53.776348"}},{"parameterName":"endTimeMin","values":{"minValue":"2022-02-23T20:48:55.393500","maxValue":"2024-11-08T22:02:14.064417"}},{"parameterName":"endTimeMax","values":{"minValue":"2022-02-23T20:48:55.393500","maxValue":"2024-11-08T22:02:14.064417"}},{"parameterName":"exposureTimeMin","values":{"minValue":0.037,"maxValue":1380.2332394366197}},{"parameterName":"exposureTimeMax","values":{"minValue":0.037,"maxValue":1380.2332394366197}},{"parameterName":"instrumentNames","values":{"categoricalValues":["VBI","CRYO-NIRSP","VISP"]}},{"parameterName":"qualityAverageFriedParameterMin","values":{"minValue":0.01695005847191685,"maxValue":2.6520787500175156e+30}},{"parameterName":"qualityAverageFriedParameterMax","values":{"minValue":0.01695005847191685,"maxValue":2.6520787500175156e+30}},{"parameterName":"qualityAveragePolarimetricAccuracyMin","values":{"minValue":0.14787299538416715,"maxValue":263.313656131021}},{"parameterName":"qualityAveragePolarimetricAccuracyMax","values":{"minValue":0.14787299538416715,"maxValue":263.313656131021}},{"parameterName":"startTimeMin","values":{"minValue":"2022-02-23T19:05:32.338002","maxValue":"2024-11-08T21:40:00.522500"}},{"parameterName":"startTimeMax","values":{"minValue":"2022-02-23T19:05:32.338002","maxValue":"2024-11-08T21:40:00.522500"}},{"parameterName":"targetTypes","values":{"categoricalValues":["coronalhole","activecorona","quietcorona","prominence","sunspot","spicules","quietsun","plages","unknown","filament"]}},{"parameterName":"averageDatasetSpectralSamplingMin","values":{"minValue":0.000540156130946172,"maxValue":0.004605031614270178}},{"parameterName":"averageDatasetSpectralSamplingMax","values":{"minValue":0.000540156130946172,"maxValue":0.004605031614270178}},{"parameterName":"averageDatasetSpatialSamplingMin","values":{"minValue":0.0,"maxValue":12388.04306084}},{"parameterName":"averageDatasetSpatialSamplingMax","values":{"minValue":0.0,"maxValue":12388.04306084}},{"parameterName":"averageDatasetTemporalSamplingMin","values":{"minValue":2.66666599999823,"maxValue":5263.145059399398}},{"parameterName":"averageDatasetTemporalSamplingMax","values":{"minValue":2.66666599999823,"maxValue":5263.145059399398}},{"parameterName":"highLevelSoftwareVersion","values":{"categoricalValues":["Alakai_10-0","Pono_6.1.5","Pono_2.1.0","Alakai_5-1","Pono_3.1.0","Pono_8.0.0","Alakai_11.1.0","Alakai_6-0","Alakai_8-0","Alakai_7-0","Pono_7.0.0","Pono_1.0.0","Alakai_3-0","Alakai_4-0","Pono_9.1.0"]}},{"parameterName":"workflowName","values":{"categoricalValues":["l0_to_l1_visp","l0_to_l1_cryonirsp_sp","l0_to_l1_cryonirsp_ci","l0_to_l1_vbi_summit-calibrated"]}},{"parameterName":"workflowVersion","values":{"categoricalValues":["1.18.6","2.16.7","1.18.0","2.0.2","1.11.1","2.20.9","2.19.5","1.2.1","2.7.5","2.19.4","2.3.1","2.19.3","1.15.0","1.0.0","1.15.1","1.3.5","1.18.4","2.16.0","1.2.0","2.10.2","2.7.4","1.18.8","2.10.3","2.20.8","2.16.4","1.1.7","2.17.0","0.16.0","1.3.2","1.17.5","1.1.10","2.15.0","2.20.7","2.7.2","2.10.15","1.4.9","1.1.2","2.7.0","2.19.1","2.20.5","2.20.10","2.10.1","2.7.3","1.12.1","2.10.12","1.1.5","1.3.3","2.6.1","2.10.13","2.19.2","2.0.1","1.18.5","1.4.8","1.8.2","2.9.0","2.10.7","2.3.0","2.10.0","1.17.1","1.13.0","2.20.6","1.14.7","1.4.11","2.20.3","2.13.1","1.4.1"]}},{"parameterName":"headerDataUnitCreationDateMin","values":{"minValue":"2022-12-08T17:30:30.462000","maxValue":"2025-01-28T00:07:38.944000"}},{"parameterName":"headerDataUnitCreationDateMax","values":{"minValue":"2022-12-08T17:30:30.462000","maxValue":"2025-01-28T00:07:38.944000"}},{"parameterName":"headerVersion","values":{"categoricalValues":["4.3.0","4.6.0","4.0.0","4.1.1","3.4.0","3.9.0","4.7.0","4.10.0","3.5.0","4.2.0","3.6.0","3.3.0","3.0.0","3.7.1","3.8.1","4.5.0"]}},{"parameterName":"spectralLines","values":{"categoricalValues":["Na I D2 (589.0 nm)","Sr I (460.73 nm)","Ca II (854.21 nm)","Fe I (630.15 nm)","Ca II K (393.37 nm)","Ca II H (396.85 nm)","Na I D1 (589.59 nm)","Mg I b1 (517.27 nm)","Fe I (630.25 nm)","Fe XIII (1079.8 nm)","Fe XIII (1074.7 nm)","H epsilon (397.0 nm)","Fe I (525.04 nm)","H beta (486.13 nm)","He I (1083.0 nm)","H alpha (656.28 nm)"]}}]}

{dkist-1.9.0 → dkist-1.9.2}/dkist/data/sample.py

@@ -4,7 +4,7 @@ This module provides some (partial) sample datasets.
 
 from ._sample import _SAMPLE_DATASETS, VISP_HEADER, _get_sample_datasets
 
-__all__ = ["download_all_sample_data", *sorted(_SAMPLE_DATASETS.keys()), "VISP_HEADER"]
+__all__ = ["download_all_sample_data", *sorted(_SAMPLE_DATASETS.keys()), "VISP_HEADER"]  # noqa: PLE0604
 
 
 # See PEP 562 (https://peps.python.org/pep-0562/) for module-level __dir__()

{dkist-1.9.0 → dkist-1.9.2}/dkist/dataset/loader.py

@@ -1,11 +1,16 @@
+import re
+import warnings
 import importlib.resources as importlib_resources
 from pathlib import Path
 from functools import singledispatch
+from collections import defaultdict
 
 from parfive import Results
 
 import asdf
 
+from dkist.utils.exceptions import DKISTUserWarning
+
 try:
     # first try to import from asdf.exceptions for asdf 2.15+
     from asdf.exceptions import ValidationError
@@ -14,6 +19,9 @@ except ImportError:
     from asdf import ValidationError
 
 
+ASDF_FILENAME_PATTERN = r"^(?P<instrument>[A-Z-]+)_L1_(?P<timestamp>\d{8}T\d{6})_(?P<datasetid>[A-Z]{5,})(?P<suffix>_user_tools|_metadata)?.asdf$"
+
+
 def asdf_open_memory_mapping_kwarg(memmap: bool) -> dict:
     if asdf.__version__ > "3.1.0":
         return {"memmap": memmap}
@@ -138,8 +146,30 @@ def _load_from_path(path: Path):
 
 def _load_from_directory(directory):
     """
-    Construct a `~dkist.dataset.Dataset` from a directory containing one
-    asdf file and a collection of FITS files.
+    Construct a `~dkist.dataset.Dataset` from a directory containing one (or
+    more) ASDF files and a collection of FITS files.
+
+    ASDF files have the generic pattern:
+
+    ``{instrument}_L1_{start_time:%Y%m%dT%H%M%S}_{dataset_id}[_{suffix}].asdf``
+
+    where the ``_{suffix}`` on the end may be absent or one of a few different
+    suffixes which have been used at different times.  When searching a
+    directory for one or more ASDF file to load we should attempt to only load
+    one per dataset ID by selecting files in suffix order.
+
+    The order of suffixes are (from newest used to oldest):
+
+    - ``_metadata``
+    - ``_user_tools``
+    - None
+
+    The algorithm used to find ASDF files to load in a directory is therefore:
+
+    - Glob the directory for all ASDF files
+    - Group all results by the filename up to and including the dataset id in the filename
+    - Ignore any ASDF files with an old suffix if a new suffix is present
+    - Throw a warning to the user if any ASDF files with older suffixes are found
     """
     base_path = Path(directory).expanduser()
     asdf_files = tuple(base_path.glob("*.asdf"))
@@ -147,12 +177,60 @@ def _load_from_directory(directory):
     if not asdf_files:
         raise ValueError(f"No asdf file found in directory {base_path}.")
 
-    if len(asdf_files) > 1:
-        return _load_from_iterable(asdf_files)
-
-    asdf_file = asdf_files[0]
+    if len(asdf_files) == 1:
+        return _load_from_asdf(asdf_files[0])
+
+    pattern = re.compile(ASDF_FILENAME_PATTERN)
+    candidates = []
+    asdfs_to_load = []
+    for filepath in asdf_files:
+        filename = filepath.name
+
+        # If the asdf file doesn't match the data center pattern then we load it
+        # as it's probably a custom user file
+        if pattern.match(filename) is None:
+            asdfs_to_load.append(filepath)
+            continue
+
+        # All the matches have to be checked
+        candidates.append(filepath)
+
+    # If we only have one match load it
+    if len(candidates) == 1:
+        asdfs_to_load += candidates
+    else:
+        # Now we group by prefix
+        matches = [pattern.match(fp.name) for fp in candidates]
+        grouped = defaultdict(list)
+        for m in matches:
+            prefix = m.string.removesuffix(".asdf").removesuffix(m.group("suffix") or "")
+            grouped[prefix].append(m.group("suffix"))
+
+        # Now we select the best suffix for each prefix
+        for prefix, suffixes in grouped.items():
+            if "_metadata" in suffixes:
+                asdfs_to_load.append(base_path / f"{prefix}_metadata.asdf")
+            elif "_user_tools" in suffixes:
+                asdfs_to_load.append(base_path / f"{prefix}_user_tools.asdf")
+            elif None in suffixes:
+                asdfs_to_load.append(base_path / f"{prefix}.asdf")
+            else:
+                # This branch should never be hit because the regex enumerates the suffixes
+                raise ValueError("Unknown suffix encountered.")  # pragma: no cover
+
+    # Throw a warning if we have skipped any files
+    if ignored_files := set(asdf_files).difference(asdfs_to_load):
+        warnings.warn(
+            f"ASDF files with old names ({', '.join([a.name for a in ignored_files])}) "
+            "were found in this directory and ignored. You may want to delete these files.",
+            DKISTUserWarning
+        )
+
+    if len(asdfs_to_load) == 1:
+        return _load_from_asdf(asdfs_to_load[0])
+
+    return _load_from_iterable(asdfs_to_load)
 
-    return _load_from_asdf(asdf_file)
 
 
 def _load_from_asdf(filepath):