dkist 1.7.0__tar.gz → 1.8.0__tar.gz

dkist-1.8.0/.circleci/codecov_upload.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import
+curl -Os https://uploader.codecov.io/latest/linux/codecov
+curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
+curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
+gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
+shasum -a 256 -c codecov.SHA256SUM
+chmod +x codecov
+./codecov "$@"

dkist-1.8.0/.circleci/config.yml

@@ -0,0 +1,113 @@
+version: 2.1
+
+no-backports: &no-backports
+  name: Skip any branches called cherry-pick
+  command: |
+    if [[ "${CIRCLE_BRANCH}" == *"cherry-pick"* || "${CIRCLE_BRANCH}" == *"backport"* ]]; then
+      circleci step halt
+    fi
+
+skip-check: &skip-check
+  name: Check for [ci skip]
+  command: bash .circleci/early_exit.sh
+
+merge-check: &merge-check
+  name: Check if we need to merge upstream main
+  command: |
+    if [[ -n "${CIRCLE_PR_NUMBER}" ]]; then
+      git fetch origin --tags
+      git fetch origin +refs/pull/$CIRCLE_PR_NUMBER/merge:pr/$CIRCLE_PR_NUMBER/merge
+      git checkout -qf pr/$CIRCLE_PR_NUMBER/merge
+    fi
+
+apt-run: &apt-install
+  name: Install apt packages
+  command: |
+    sudo apt update
+    sudo apt install -y libopenjp2-7
+
+jobs:
+  figure:
+    parameters:
+      jobname:
+        type: string
+    docker:
+      - image: cimg/python:3.12
+    environment:
+      TOXENV=<< parameters.jobname >>
+    steps:
+      - run: *no-backports
+      - checkout
+      - run: *skip-check
+      - run: *merge-check
+      - run: *apt-install
+      - run: pip install --user -U tox tox-pypi-filter
+      - run: tox -v
+      - run:
+          name: Running codecov
+          command: bash -e .circleci/codecov_upload.sh -f ".tmp/${TOXENV}/coverage.xml"
+      - store_artifacts:
+          path: .tmp/<< parameters.jobname >>/figure_test_images
+
+  deploy-reference-images:
+    parameters:
+      jobname:
+        type: string
+    docker:
+      - image: cimg/python:3.12
+    environment:
+      TOXENV: << parameters.jobname >>
+      GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa_7b8fc81c13a3b446ec9aa50d3f626978
+    steps:
+      - checkout
+      - run: *skip-check
+      - run: *merge-check
+      - run: *apt-install
+      # Clear out all the ssh keys so that it always uses the write deploy key
+      - run: ssh-add -D
+      # Add private key for deploying to the figure tests repo
+      - add_ssh_keys:
+          fingerprints: "SHA256:vf80el6ZY/FiLKo+eXblG/DQfCuJBwrQvH1S37vjG5I"
+      - run: ssh-keyscan github.com >> ~/.ssh/known_hosts
+      - run: git config --global user.email "dkist@circleci" && git config --global user.name "DKIST Circle CI"
+      - run: git clone git@github.com:DKISTDC/dkist-figure-tests.git --depth 1 -b dkist-${CIRCLE_BRANCH} ~/dkist-figure-tests/
+      # Generate Reference images
+      - run: pip install --user -U tox tox-pypi-filter
+      - run: rm -rf /home/circleci/dkist-figure-tests/figures/$TOXENV/*
+      - run: tox -v -- --mpl-generate-path=/home/circleci/dkist-figure-tests/figures/$TOXENV | tee toxlog
+      - run: |
+          hashlib=$(grep "^figure_hashes.*\.json$" toxlog)
+          cp ./dkist/tests/$hashlib /home/circleci/dkist-figure-tests/figures/$TOXENV/
+      - run: |
+          cd ~/dkist-figure-tests/
+          git pull
+          git status
+          git add .
+          git commit -m "Update reference figures from ${CIRCLE_BRANCH}" || echo "No changes to reference images to deploy"
+          git push
+
+workflows:
+  version: 2
+
+  figure-tests:
+    jobs:
+      - figure:
+          name: << matrix.jobname >>
+          matrix:
+            parameters:
+              jobname:
+                - "py312-figure"
+
+      - deploy-reference-images:
+          name: baseline-<< matrix.jobname >>
+          matrix:
+            parameters:
+              jobname:
+                - "py312-figure"
+          requires:
+            - << matrix.jobname >>
+          filters:
+            branches:
+              only:
+                - main
+                - figuretests

dkist-1.8.0/.circleci/early_exit.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+commitmessage=$(git log --pretty=%B -n 1)
+if [[ $commitmessage = *"[ci skip]"* ]] || [[ $commitmessage = *"[skip ci]"* ]]; then
+    echo "Skipping build because [ci skip] found in commit message"
+    circleci step halt
+fi

{dkist-1.7.0 → dkist-1.8.0}/.cruft.json

@@ -1,6 +1,6 @@
 {
   "template": "https://github.com/sunpy/package-template",
-  "commit": "4721cdbdc9577fbf66f5e0f30fd4133a7723aeb9",
+  "commit": "f7458b35be5824d419efd6ce8c135ff67a00d1d5",
   "checkout": null,
   "context": {
     "cookiecutter": {

{dkist-1.7.0 → dkist-1.8.0}/.github/workflows/codspeed.yml

@@ -16,8 +16,9 @@ jobs:
         with:
           python-version: '3.12'
       - run: python -m pip install .[tests] pytest-codspeed 'numpy<2'
+      - run: python -c "from dkist.data.sample import download_all_sample_data; download_all_sample_data()"
       - name: Run benchmarks
-        uses: CodspeedHQ/action@v2
+        uses: CodspeedHQ/action@v3
         with:
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: "pytest -vvv -r fEs --pyargs dkist --codspeed"

{dkist-1.7.0 → dkist-1.8.0}/.github/workflows/main.yml

@@ -83,13 +83,18 @@ jobs:
 
   publish:
     needs: [tests]
-    if: github.event_name != 'pull_request'
+    if: |
+      github.event_name != 'pull_request' || (
+        github.event_name == 'pull_request' &&
+        contains(github.event.pull_request.labels.*.name, 'Run publish')
+      )
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@main
     with:
       python-version: '3.10'
       test_extras: tests
-      test_command: pytest --pyargs dkist
-      upload_to_pypi: ${{ startsWith(github.event.ref, 'refs/tags/v') && !endsWith(github.event.ref, '.dev') }}
+      test_command: pytest --pyargs dkist -k "not test_fail"
+      # We have to work around a github runner bug here: https://github.com/actions/runner/issues/2788#issuecomment-2145922705
+      upload_to_pypi: ${{ startsWith(github.ref || format('{0}{1}', 'refs/tags/', github.event.release.tag_name), 'refs/tags/v') && !endsWith(github.ref || format('{0}{1}', 'refs/tags/', github.event.release.tag_name), '.dev') }}
     secrets:
       pypi_token: ${{ secrets.PYPI_API_TOKEN }}
 

{dkist-1.7.0 → dkist-1.8.0}/.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
     # This should be before any formatting hooks like isort
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.5.0"
+    rev: "v0.6.2"
     hooks:
       - id: ruff
         args: ["--fix"]

dkist-1.8.0/.readthedocs.yaml

@@ -0,0 +1,33 @@
+version: 2
+
+build:
+  os: ubuntu-lts-latest
+  tools:
+    python: "mambaforge-latest"
+  jobs:
+    post_checkout:
+      - git fetch --unshallow || true
+    pre_install:
+      - git update-index --assume-unchanged .rtd-environment.yml docs/conf.py
+    pre_build:
+      # Note $DKIST_SAMPLE_DIR is set by RTD to ~/dkist_data
+      - python -c "from dkist.data.sample import download_all_sample_data; download_all_sample_data()"
+      - python -c "from dkist.data.sample import VISP_BKPLX; print(VISP_BKPLX)"
+
+conda:
+  environment: .rtd-environment.yml
+
+sphinx:
+  builder: html
+  configuration: docs/conf.py
+  fail_on_warning: false
+
+formats:
+  - htmlzip
+
+python:
+  install:
+    - method: pip
+      extra_requirements:
+        - docs
+      path: .

{dkist-1.7.0 → dkist-1.8.0}/.rtd-environment.yml

@@ -2,6 +2,6 @@ name: dkist
 channels:
   - conda-forge
 dependencies:
-  - python=3.10
+  - python=3.12
   - pip
   - graphviz!=2.42.*,!=2.43.*

{dkist-1.7.0 → dkist-1.8.0}/CHANGELOG.rst

@@ -1,3 +1,28 @@
+1.8.0 (2024-09-04)
+==================
+
+Features
+--------
+
+- Add various features for easier inspection of `TiledDataset`:
+  - `__repr__` method to output basic dataset info;
+  - `tiles_shape` property to access data array shape for each individual tile;
+  - `slice_tiles()` method to apply the same slice to all datasets. (`#402 <https://github.com/DKISTDC/dkist/pull/402>`_)
+- Add `TiledDataset.plot()` quicklook method. (`#408 <https://github.com/DKISTDC/dkist/pull/408>`_)
+- Add CircleCI config to enable figure comparison testing with pytest_mpl. (`#415 <https://github.com/DKISTDC/dkist/pull/415>`_)
+
+
+Trivial/Internal Changes
+------------------------
+
+- Add some more benchmarks to track performance of more parts of the user tools. (`#387 <https://github.com/DKISTDC/dkist/pull/387>`_)
+- Add benchmarks for slicing and computing datasets. (`#394 <https://github.com/DKISTDC/dkist/pull/394>`_)
+- Added two partial datasets to `dkist.data.sample` for documentation and testing. (`#421 <https://github.com/DKISTDC/dkist/pull/421>`_)
+- replace usages of ``copy_arrays`` with ``memmap`` for ``asdf>=3.1.0`` (`#422 <https://github.com/DKISTDC/dkist/pull/422>`_)
+- Update Dataset representation for better readability. (`#431 <https://github.com/DKISTDC/dkist/pull/431>`_)
+- SKip a failing test in the release publish build. (`#434 <https://github.com/DKISTDC/dkist/pull/434>`_)
+
+
 1.7.0 (2024-07-11)
 ==================
 

{dkist-1.7.0 → dkist-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dkist
-Version: 1.7.0
+Version: 1.8.0
 Summary: DKIST User Tools
 Author-email: NSO / AURA <stuart@cadair.com>
 Project-URL: repository, https://github.com/DKISTDC/dkist
@@ -107,3 +107,12 @@ Code of Conduct
 ---------------
 
 When you are interacting with the `dkist` project you are asked to follow the SunPy `Code of Conduct <https://sunpy.org/coc>`__.
+
+Releasing The `dkist` package
+-----------------------------
+
+Doing a release of this package is automated using GitHub actions, the release can be done following these steps:
+
+1. Run the pre-release workflow on GitHub actions, this can be done by clicking "run workflow" `here <https://github.com/DKISTDC/dkist/actions/workflows/prepare-release.yml>`__.
+2. That workflow will have opened a PR, you should review it and merge it as normal.
+3. That workflow will also have drafted a GitHub release `here <https://github.com/DKISTDC/dkist/releases>`__, once the PR is merged you can publish that release, this will trigger the rest as long as the CI passes.

{dkist-1.7.0 → dkist-1.8.0}/README.rst

@@ -52,3 +52,12 @@ Code of Conduct
 ---------------
 
 When you are interacting with the `dkist` project you are asked to follow the SunPy `Code of Conduct <https://sunpy.org/coc>`__.
+
+Releasing The `dkist` package
+-----------------------------
+
+Doing a release of this package is automated using GitHub actions, the release can be done following these steps:
+
+1. Run the pre-release workflow on GitHub actions, this can be done by clicking "run workflow" `here <https://github.com/DKISTDC/dkist/actions/workflows/prepare-release.yml>`__.
+2. That workflow will have opened a PR, you should review it and merge it as normal.
+3. That workflow will also have drafted a GitHub release `here <https://github.com/DKISTDC/dkist/releases>`__, once the PR is merged you can publish that release, this will trigger the rest as long as the CI passes.

{dkist-1.7.0 → dkist-1.8.0}/dkist/__init__.py

@@ -4,7 +4,10 @@ The DKIST package aims to help you search, obtain and use DKIST data as part of
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version as _version
 
+import platformdirs as _platformdirs
+
 from .logger import setup_default_dkist_logger as _setup_log
+import dkist.config as _config
 
 log = _setup_log(__name__)
 
@@ -14,7 +17,7 @@ except PackageNotFoundError:
     __version__ = "unknown"
 
 
-__all__ = ["TiledDataset", "Dataset", "load_dataset", "system_info"]
+__all__ = ["TiledDataset", "Dataset", "load_dataset", "system_info", "conf"]
 
 
 def write_default_config(overwrite=False):
@@ -29,6 +32,19 @@ def write_default_config(overwrite=False):
     return _config.create_config_file("dkist", "dkist", overwrite=overwrite)
 
 
+class Conf(_config.ConfigNamespace):
+    """
+    Configuration Parameters for the `dkist` Package.
+    """
+    sample_data_directory = _config.ConfigItem(
+        _platformdirs.user_data_dir(appname="dkist"),
+        "Location to download sample data to."
+    )
+
+
+conf = Conf()
+
+
 # Do internal imports last (so logger etc is initialised)
 from dkist.dataset import Dataset, TiledDataset, load_dataset
 from dkist.utils.sysinfo import system_info

{dkist-1.7.0 → dkist-1.8.0}/dkist/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.7.0'
-__version_tuple__ = version_tuple = (1, 7, 0)
+__version__ = version = '1.8.0'
+__version_tuple__ = version_tuple = (1, 8, 0)

{dkist-1.7.0 → dkist-1.8.0}/dkist/conftest.py

@@ -282,6 +282,15 @@ def simple_tiled_dataset(dataset):
     return TiledDataset(dataset_array, dataset.meta["inventory"])
 
 
+@pytest.fixture
+def large_tiled_dataset(tmp_path_factory):
+    vbidir = tmp_path_factory.mktemp("data")
+    with gzip.open(Path(rootdir) / "large_vbi.asdf.gz", mode="rb") as gfo:
+        with open(vbidir / "test_vbi.asdf", mode="wb") as afo:
+            afo.write(gfo.read())
+    return load_dataset(vbidir / "test_vbi.asdf")
+
+
 @pytest.fixture
 def small_visp_dataset():
     """

dkist-1.8.0/dkist/data/_sample.py

@@ -0,0 +1,110 @@
+import os
+import tarfile
+from pathlib import Path
+from urllib.parse import urljoin
+
+from parfive import Downloader, Results
+
+from astropy.io import fits
+
+from dkist import conf
+
+VISP_HEADER = fits.Header.fromtextfile(Path(__file__).parent / "VISP_HEADER.hdr")
+_SAMPLE_DATASETS = {
+    "VISP_BKPLX": ("https://g-a36282.cd214.a567.data.globus.org/user_tools_tutorial_data/", "BKPLX_stokesI.tar"),
+    "VBI_AJQWW": ("https://g-a36282.cd214.a567.data.globus.org/user_tools_tutorial_data/", "AJQWW_single_mosaic.tar"),
+}
+
+
+def _download_and_extract_sample_data(names, overwrite, path):
+    """
+    Downloads a list of files.
+
+    Parameters
+    ----------
+    names : list[str]
+        The names of the datasets to download and extract
+    overwrite : bool
+        Will overwrite a file on disk if True.
+    path : `pathlib.Path`
+        The sample data path to save the tar files
+
+    Returns
+    -------
+    `parfive.Results`
+        Download results. Will behave like a list of files.
+    """
+    dl = Downloader(overwrite=overwrite, progress=True)
+
+    existing_files = []
+
+    for name in names:
+        base_url, filename = _SAMPLE_DATASETS[name]
+        if (filepath := path / filename).exists():
+            existing_files.append(filepath)
+            continue
+
+        url = urljoin(base_url, filename)
+        dl.enqueue_file(url, path=path)
+
+    results = Results()
+    if dl.queued_downloads:
+        results = dl.download()
+    results += existing_files
+
+    file_folder = {filename: name for name, (_, filename) in _SAMPLE_DATASETS.items() if name in names}
+
+    for i, tarpath in enumerate(results):
+        output_path = path / file_folder[Path(tarpath).name]
+        with tarfile.open(tarpath, "r:*") as tar:
+            tar.extractall(path=output_path, filter=tarfile.fully_trusted_filter)
+        results[i] = output_path
+
+    return results
+
+
+def _get_sample_datasets(dataset_names, no_download=False, force_download=False):
+    """
+    Returns a list of disk locations corresponding to a list of filenames for
+    sample data, downloading the sample data files as necessary.
+
+    Parameters
+    ----------
+    no_download : `bool`
+        If ``True``, do not download any files, even if they are not present.
+        Default is ``False``.
+    force_download : `bool`
+        If ``True``, download all files, and overwrite any existing ones.
+        Default is ``False``.
+
+    Returns
+    -------
+    `list` of `pathlib.Path`
+        List of disk locations corresponding to the list of filenames.  An entry
+        will be ``None`` if ``no_download == True`` and the file is not present.
+
+    Raises
+    ------
+    RuntimeError
+        Raised if any of the files cannot be downloaded from any of the mirrors.
+    """
+    sampledata_dir = Path(conf.sample_data_directory)
+    if env_override := os.environ.get("DKIST_SAMPLE_DIR"):
+        # For some reason, RTD adds ' around the path in the env var.
+        sampledata_dir = Path(env_override.strip("'"))
+    sampledata_dir = sampledata_dir.expanduser()
+
+    datasets = dict((k,v) for k, v in _SAMPLE_DATASETS.items() if k in dataset_names)  # noqa: C402
+    download_paths = [sampledata_dir / fn for _, fn in datasets.values()]
+
+    if no_download:
+        return [sampledata_dir / name for name in datasets.keys() if (sampledata_dir / name).exists()]
+
+    results = _download_and_extract_sample_data(datasets.keys(), overwrite=force_download, path=sampledata_dir)
+
+    if results.errors:
+        raise RuntimeError(
+            f"{len(results.errors)} sample data files failed "
+            "to download, the first error is above.") from results.errors[0].exception
+
+    return list(results)

dkist-1.8.0/dkist/data/api_search_values.json

@@ -0,0 +1 @@
+{"parameterValues":[{"parameterName":"createDateMin","values":{"minValue":"2022-12-08T19:07:55.038280","maxValue":"2024-09-02T20:19:56.988017"}},{"parameterName":"createDateMax","values":{"minValue":"2022-12-08T19:07:55.038280","maxValue":"2024-09-02T20:19:56.988017"}},{"parameterName":"endTimeMin","values":{"minValue":"2022-02-23T20:48:55.393500","maxValue":"2024-08-27T21:45:38.497106"}},{"parameterName":"endTimeMax","values":{"minValue":"2022-02-23T20:48:55.393500","maxValue":"2024-08-27T21:45:38.497106"}},{"parameterName":"exposureTimeMin","values":{"minValue":0.037,"maxValue":1380.2332394366197}},{"parameterName":"exposureTimeMax","values":{"minValue":0.037,"maxValue":1380.2332394366197}},{"parameterName":"instrumentNames","values":{"categoricalValues":["CRYO-NIRSP","VBI","VISP"]}},{"parameterName":"qualityAverageFriedParameterMin","values":{"minValue":0.01695005847191685,"maxValue":2.6520787500175156e+30}},{"parameterName":"qualityAverageFriedParameterMax","values":{"minValue":0.01695005847191685,"maxValue":2.6520787500175156e+30}},{"parameterName":"qualityAveragePolarimetricAccuracyMin","values":{"minValue":0.14787299538416715,"maxValue":263.313656131021}},{"parameterName":"qualityAveragePolarimetricAccuracyMax","values":{"minValue":0.14787299538416715,"maxValue":263.313656131021}},{"parameterName":"startTimeMin","values":{"minValue":"2022-02-23T19:05:32.338002","maxValue":"2024-08-27T20:09:46.569864"}},{"parameterName":"startTimeMax","values":{"minValue":"2022-02-23T19:05:32.338002","maxValue":"2024-08-27T20:09:46.569864"}},{"parameterName":"targetTypes","values":{"categoricalValues":["coronalhole","activecorona","quietcorona","prominence","sunspot","spicules","quietsun","plages","unknown","filament"]}},{"parameterName":"averageDatasetSpectralSamplingMin","values":{"minValue":0.000540156130946172,"maxValue":0.004605031614270178}},{"parameterName":"averageDatasetSpectralSamplingMax","values":{"minValue":0.000540156130946172,"maxValue":0.004605031614270178}},{"parameterName":"averageDatasetSpatialSamplingMin","values":{"minValue":0.0,"maxValue":12388.04306084}},{"parameterName":"averageDatasetSpatialSamplingMax","values":{"minValue":0.0,"maxValue":12388.04306084}},{"parameterName":"averageDatasetTemporalSamplingMin","values":{"minValue":7.403500000002339,"maxValue":5263.145059399399}},{"parameterName":"averageDatasetTemporalSamplingMax","values":{"minValue":7.403500000002339,"maxValue":5263.145059399399}},{"parameterName":"highLevelSoftwareVersion","values":{"categoricalValues":["Alakai_10-0","Pono_6.1.5","Pono_2.1.0","Alakai_5-1","Pono_3.1.0","Pono_8.0.0","Alakai_11.1.0","Alakai_6-0","Alakai_8-0","Alakai_7-0","Pono_7.0.0","Pono_1.0.0","Alakai_3-0","Alakai_4-0"]}},{"parameterName":"workflowName","values":{"categoricalValues":["l0_to_l1_visp","l0_to_l1_cryonirsp_sp","l0_to_l1_cryonirsp_ci","l0_to_l1_vbi_summit-calibrated"]}},{"parameterName":"workflowVersion","values":{"categoricalValues":["2.7.4","2.16.4","2.10.3","1.1.7","0.16.0","2.16.7","1.1.10","2.15.0","2.7.2","2.10.15","2.7.0","2.10.1","2.0.2","2.7.3","1.12.1","2.10.12","1.1.5","1.11.1","2.6.1","1.2.1","2.10.13","2.7.5","2.0.1","1.4.8","1.8.2","2.9.0","2.10.7","2.3.1","2.3.0","2.10.0","1.0.0","1.13.0","1.14.7","1.4.11","1.4.1","2.16.0","2.13.1","1.2.0","2.10.2"]}},{"parameterName":"headerDataUnitCreationDateMin","values":{"minValue":"2022-12-08T17:25:51.965000","maxValue":"2024-09-02T16:39:39.678000"}},{"parameterName":"headerDataUnitCreationDateMax","values":{"minValue":"2022-12-08T17:25:51.965000","maxValue":"2024-09-02T16:39:39.678000"}},{"parameterName":"headerVersion","values":{"categoricalValues":["4.3.0","4.6.0","4.0.0","4.1.1","3.4.0","3.9.0","3.5.0","4.2.0","3.6.0","3.3.0","3.0.0","3.7.1","3.8.1","4.5.0"]}}]}

dkist-1.8.0/dkist/data/sample.py

@@ -0,0 +1,27 @@
+"""
+This module provides some (partial) sample datasets.
+"""
+
+from ._sample import _SAMPLE_DATASETS, VISP_HEADER, _get_sample_datasets
+
+__all__ = ["download_all_sample_data", *sorted(_SAMPLE_DATASETS.keys()), "VISP_HEADER"]
+
+
+# See PEP 562 (https://peps.python.org/pep-0562/) for module-level __dir__()
+def __dir__():
+    return __all__
+
+
+# See PEP 562 (https://peps.python.org/pep-0562/) for module-level __getattr__()
+def __getattr__(name):
+    if name in _SAMPLE_DATASETS:
+        return _get_sample_datasets(name)[0]
+
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+def download_all_sample_data():
+    """
+    Download all sample data at once that has not already been downloaded.
+    """
+    return _get_sample_datasets(_SAMPLE_DATASETS.keys())

dkist-1.8.0/dkist/data/tests/test_sample.py

@@ -0,0 +1,38 @@
+import os
+from unittest.mock import call
+
+import pytest
+
+
+@pytest.fixture
+def tmp_sample_dir(tmp_path):
+    old_path = os.environ.get("DKIST_SAMPLE_DIR", "")
+    os.environ["DKIST_SAMPLE_DIR"] = str(tmp_path)
+    yield tmp_path
+    os.environ["DKIST_SAMPLE_DIR"] = old_path
+
+
+def test_module_dir():
+    import dkist.data.sample
+
+    assert "VBI_AJQWW" in dir(dkist.data.sample)
+    assert "VISP_BKPLX" in dir(dkist.data.sample)
+
+
+@pytest.mark.parametrize("attrname", ["VBI_AJQWW", "VISP_BKPLX"])
+def test_module_getattr(mocker, attrname):
+    mock = mocker.patch("dkist.data.sample._get_sample_datasets")
+    import dkist.data.sample
+
+    getattr(dkist.data.sample, attrname)
+
+    mock.assert_has_calls([call(attrname), call().__getitem__(0)])
+
+
+@pytest.mark.internet_off
+def test_fail(tmp_sample_dir):
+    """
+    No remote data means this test should fail.
+    """
+    with pytest.raises(RuntimeError, match="1 sample data files failed"):
+        from dkist.data.sample import VISP_BKPLX  # noqa: F401

{dkist-1.7.0 → dkist-1.8.0}/dkist/dataset/loader.py

@@ -14,6 +14,12 @@ except ImportError:
     from asdf import ValidationError
 
 
+def asdf_open_memory_mapping_kwarg(memmap: bool) -> dict:
+    if asdf.__version__ > "3.1.0":
+        return {"memmap": memmap}
+    return {"copy_arrays": not memmap}
+
+
 @singledispatch
 def load_dataset(target):
     """
@@ -39,45 +45,48 @@ def load_dataset(target):
     Examples
     --------
 
+    >>> import dkist
+
     >>> dkist.load_dataset("/path/to/VISP_L1_ABCDE.asdf")  # doctest: +SKIP
 
     >>> dkist.load_dataset("/path/to/ABCDE/")  # doctest: +SKIP
 
     >>> dkist.load_dataset(Path("/path/to/ABCDE"))  # doctest: +SKIP
 
-    >>> from sunpy.net import Fido, attrs as a
-    >>> import dkist.net
-    >>> search_results = Fido.search(a.dkist.Dataset("AGLKO"))   # doctest: +REMOTE_DATA
-    >>> files = Fido.fetch(search_results)   # doctest: +REMOTE_DATA
-    >>> dkist.load_dataset(files)   # doctest: +REMOTE_DATA
-    <dkist.dataset.dataset.Dataset object at ...>
-    This Dataset has 4 pixel and 5 world dimensions
+    >>> from dkist.data.sample import VISP_BKPLX  # doctest: +REMOTE_DATA
+    >>> print(dkist.load_dataset(VISP_BKPLX))  # doctest: +REMOTE_DATA
+    This VISP Dataset BKPLX consists of 1700 frames.
+    Files are stored in ...VISP_BKPLX
     <BLANKLINE>
-    dask.array<reshape, shape=(4, 1000, 976, 2555), dtype=float64, chunksize=(1, 1, 976, 2555), chunktype=numpy.ndarray>
+    This Dataset has 4 pixel and 5 world dimensions.
     <BLANKLINE>
-    Pixel Dim  Axis Name                Data size  Bounds
+    The data are represented by a <class 'dask.array.core.Array'> object:
+    dask.array<reshape, shape=(4, 425, 980, 2554), dtype=float64, chunksize=(1, 1, 980, 2554), chunktype=numpy.ndarray>
+    <BLANKLINE>
+    Array Dim  Axis Name                Data size  Bounds
             0  polarization state               4  None
-            1  raster scan step number       1000  None
-            2  dispersion axis                976  None
-            3  spatial along slit            2555  None
+            1  raster scan step number        425  None
+            2  dispersion axis                980  None
+            3  spatial along slit            2554  None
     <BLANKLINE>
     World Dim  Axis Name                  Physical Type                   Units
-            0  stokes                     phys.polarization.stokes        unknown
-            1  time                       time                            s
+            4  stokes                     phys.polarization.stokes        unknown
+            3  time                       time                            s
             2  helioprojective latitude   custom:pos.helioprojective.lat  arcsec
-            3  wavelength                 em.wl                           nm
-            4  helioprojective longitude  custom:pos.helioprojective.lon  arcsec
+            1  wavelength                 em.wl                           nm
+            0  helioprojective longitude  custom:pos.helioprojective.lon  arcsec
     <BLANKLINE>
     Correlation between pixel and world axes:
     <BLANKLINE>
-                   Pixel Dim
-    World Dim    0    1    2    3
-            0  yes   no   no   no
-            1   no  yes   no   no
-            2   no  yes   no  yes
-            3   no   no  yes   no
-            4   no  yes   no  yes
-
+                              |                      PIXEL DIMENSIONS
+                              |   spatial    |  dispersion  | raster scan  | polarization
+             WORLD DIMENSIONS |  along slit  |     axis     | step number  |    state
+    ------------------------- | ------------ | ------------ | ------------ | ------------
+    helioprojective longitude |      x       |              |      x       |
+                   wavelength |              |      x       |              |
+     helioprojective latitude |      x       |              |      x       |
+                         time |              |              |      x       |
+                       stokes |              |              |              |      x
     """
     known_types = _known_types_docs().keys()
     raise TypeError(f"Input type {type(target).__name__} not recognised. It must be one of {', '.join(known_types)}.")
@@ -156,7 +165,7 @@ def _load_from_asdf(filepath):
     try:
         with importlib_resources.as_file(importlib_resources.files("dkist.io") / "level_1_dataset_schema.yaml") as schema_path:
             with asdf.open(filepath, custom_schema=schema_path.as_posix(),
-                           lazy_load=False, copy_arrays=True) as ff:
+                           lazy_load=False, **asdf_open_memory_mapping_kwarg(memmap=False)) as ff:
                 ds = ff.tree["dataset"]
                 if isinstance(ds, TiledDataset):
                     for sub in ds.flat: