essreduce 25.9.0__py3-none-any.whl → 25.10.0__py3-none-any.whl

ess/reduce/data/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2025 Scipp contributors (https://github.com/scipp)
+"""Data files bundled with ESSreduce.
+
+This module requires the Pooch package which is not a hard dependency of ESSreduce.
+It has to be installed separately with either pip or conda.
+"""
+
+from ._registry import Entry, LocalRegistry, PoochRegistry, Registry, make_registry
+
+__all__ = [
+    'Entry',
+    'LocalRegistry',
+    'PoochRegistry',
+    'Registry',
+    'make_registry',
+]

ess/reduce/data/_registry.py

@@ -0,0 +1,355 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2025 Scipp contributors (https://github.com/scipp)
+
+from __future__ import annotations
+
+import hashlib
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+from typing import Any
+
+_LOCAL_CACHE_ENV_VAR = "SCIPP_DATA_DIR"
+_LOCAL_REGISTRY_ENV_VAR = "SCIPP_OVERRIDE_DATA_DIR"
+
+
+def make_registry(
+    prefix: str,
+    files: Mapping[str, str | Entry],
+    *,
+    version: str,
+    base_url: str = "https://public.esss.dk/groups/scipp",
+    retry_if_failed: int = 3,
+) -> Registry:
+    """Create a file registry object.
+
+    By default, this function creates a :class:`PoochRegistry` to download files
+    via HTTP from an online file store.
+    This can be overridden by setting the environment variable `SCIPP_DATA_DIR` to a
+    path on the local file system.
+    In this case, a :class:`LocalRegistry` is returned.
+
+    Files are specified as a dict using either the Pooch string format explicitly
+    constructed :class:`Entry` objects:
+
+        >>> from ess.reduce.data import Entry
+        >>> files = {
+        ...    "file1.dat": "md5:1234567890abcdef",
+        ...    "file2.csv": Entry(alg="md5", chk="abcdef123456789"),
+        ...    "folder/nested.dat": "blake2b:1234567890abcdef",
+        ...    "zipped.zip": Entry(alg="blake2b", chk="abcdef123456789", unzip=True),
+        ... }
+
+    In the example above, the specifications for ``file1.dat`` and ``file2.csv`` are
+    essentially equivalent.
+    ``folder/nested.dat`` is a file in a subfolder.
+    Paths like this must always use forward slashes (/) even on Windows.
+
+    As shown above, it is possible to automatically unzip
+    files by specifying ``unzip=True``.
+    When calling ``registry.get_path("zipped.zip")`` the file will be unzipped and
+    a path to the content is returned.
+    This expects that there is only a single file in the zip archive.
+
+    The complete path to the source file is constructed as follows:
+
+    - Pooch: ``{base_url}/{prefix}/{version}/{name}``
+    - Local: ``{SCIPP_OVERRIDE_DATA_DIR}/{prefix}/{version}/{name}``
+
+    When using Pooch, files are downloaded to the user's cache directory.
+    This can be controlled with the environment variable ``SCIPP_CACHE_DIR``.
+
+    Parameters
+    ----------
+    prefix:
+        Prefix to add to all file names.
+    files:
+        Mapping of file names to checksums or :class:`Entry` objects.
+    version:
+        A version string for the files.
+    base_url:
+        URL for the online file store.
+        Ignored if the override environment variable is set.
+    retry_if_failed:
+        Number of retries when downloading a file.
+        Ignored if the override environment variable is set.
+
+    Returns
+    -------
+    :
+        Either a :class:`PoochRegistry` or :class:`LocalRegistry`.
+    """
+    if (override := os.environ.get(_LOCAL_REGISTRY_ENV_VAR)) is not None:
+        return LocalRegistry(
+            _check_local_override_path(override),
+            prefix,
+            files,
+            version=version,
+            base_url=base_url,
+            retry_if_failed=retry_if_failed,
+        )
+    return PoochRegistry(
+        prefix,
+        files,
+        version=version,
+        base_url=base_url,
+        retry_if_failed=retry_if_failed,
+    )
+
+
+def _check_local_override_path(override: str) -> Path:
+    path = Path(override)
+    if not path.is_dir():
+        raise ValueError(
+            f"The data override path '{override}' is not a directory. If you want to "
+            "download files instead, unset the environment variable "
+            f"{_LOCAL_REGISTRY_ENV_VAR}."
+        )
+    return path
+
+
+@dataclass(frozen=True, slots=True)
+class Entry:
+    """An entry in a registry."""
+
+    chk: str
+    """Checksum."""
+    alg: str
+    """Checksum algorithm."""
+    unzip: bool = False
+    """Whether to unzip the file."""
+
+    @classmethod
+    def from_pooch_string(cls, pooch_string: str) -> Entry:
+        alg, chk = pooch_string.split(":")
+        return cls(chk=chk, alg=alg)
+
+
+class Registry(ABC):
+    def __init__(self, files: Mapping[str, str | Entry]) -> None:
+        self._files = _to_file_entries(files)
+
+    @abstractmethod
+    def get_path(self, name: str) -> Path:
+        """Get the path to a file in the registry.
+
+        Depending on the implementation, the file is downloaded if necessary.
+
+        Note that implementations are allowed to cache return values of this method
+        to avoid recomputing potentially expensive checksums.
+        This usually means that the ``Registry`` object itself gets stored until the
+        Python interpreter shuts down.
+        However, registries are small and do not own resources.
+
+        Parameters
+        ----------
+        name:
+            Name of the file to get the path for.
+
+        Returns
+        -------
+        :
+            The Path to the file.
+        """
+
+    def _needs_unzip(self, name: str) -> bool:
+        return self._files[name].unzip
+
+
+class PoochRegistry(Registry):
+    def __init__(
+        self,
+        prefix: str,
+        files: Mapping[str, str | Entry],
+        *,
+        version: str,
+        base_url: str,
+        retry_if_failed: int = 3,
+    ) -> None:
+        self._registry = _create_pooch(
+            prefix,
+            files,
+            version=version,
+            base_url=base_url,
+            retry_if_failed=retry_if_failed,
+        )
+        self._unzip_processor = _import_pooch().Unzip()
+        super().__init__(files)
+
+    @cache  # noqa: B019
+    def get_path(self, name: str) -> Path:
+        """Get the path to a file in the registry.
+
+        Downloads the file if necessary.
+        """
+        if self._needs_unzip(name):
+            paths: list[str] = self._registry.fetch(  # type: ignore[assignment]
+                name, processor=self._unzip_processor
+            )
+            return Path(_expect_single_unzipped(paths, name))
+        return Path(self._registry.fetch(name))
+
+
+class LocalRegistry(Registry):
+    def __init__(
+        self,
+        source_path: Path,
+        prefix: str,
+        files: Mapping[str, str | Entry],
+        *,
+        version: str,
+        base_url: str,
+        retry_if_failed: int = 3,
+    ) -> None:
+        # Piggyback off of Pooch to determine the cache directory.
+        pooch_registry = _create_pooch(
+            prefix,
+            files,
+            version=version,
+            base_url=base_url,
+            retry_if_failed=retry_if_failed,
+        )
+        pooch = _import_pooch()
+        self._unzip_processor = pooch.processors.Unzip(extract_dir=pooch_registry.path)
+        self._source_path = source_path.resolve().joinpath(*prefix.split("/"), version)
+        super().__init__(files)
+
+    @cache  # noqa: B019
+    def get_path(self, name: str) -> Path:
+        """Get the path to a file in the registry."""
+        try:
+            entry = self._files[name]
+        except KeyError:
+            raise ValueError(f"File '{name}' is not in the registry.") from None
+
+        path = self._local_path(name)
+        if not path.exists():
+            raise FileNotFoundError(
+                f"File '{name}' is registered but does not exist on the file system. "
+                f"Expected it at '{path}'."
+            )
+
+        _check_hash(name, path, entry)
+
+        if self._needs_unzip(name):
+            return Path(
+                _expect_single_unzipped(
+                    self._unzip_processor(os.fspath(path), "download", None), path
+                )
+            )
+        return path
+
+    def _local_path(self, name: str) -> Path:
+        # Split on "/" because `name` is always a POSIX-style path, but the return
+        # value is a system path, i.e., it can be a Windows-style path.
+        return self._source_path.joinpath(*name.split("/"))
+
+
+def _import_pooch() -> Any:
+    try:
+        import pooch
+    except ImportError:
+        raise ImportError(
+            "You need to install Pooch to access test and tutorial files. "
+            "See https://www.fatiando.org/pooch/latest/index.html"
+        ) from None
+
+    return pooch
+
+
+def _create_pooch(
+    prefix: str,
+    files: Mapping[str, str | Entry],
+    *,
+    version: str,
+    base_url: str,
+    retry_if_failed: int = 3,
+) -> Any:
+    pooch = _import_pooch()
+    return pooch.create(
+        path=pooch.os_cache(prefix),
+        env=_LOCAL_CACHE_ENV_VAR,
+        base_url=f'{base_url}/{prefix}/{version}/',
+        registry=_to_pooch_registry(files),
+        retry_if_failed=retry_if_failed,
+    )
+
+
+def _pooch_unzip_processor(extract_dir: Path) -> Any:
+    try:
+        import pooch
+    except ImportError:
+        raise ImportError("You need to install Pooch to unzip files.") from None
+
+    return pooch.processors.Unzip(extract_dir=os.fspath(extract_dir))
+
+
+def _expect_single_unzipped(paths: list[str], archive: str | os.PathLike) -> str:
+    if len(paths) != 1:
+        raise ValueError(
+            f"Expected exactly one file to unzip, got {len(paths)} in "
+            f"'{os.fspath(archive)}'."
+        )
+    return paths[0]
+
+
+def _check_hash(name: str, path: Path, entry: Entry) -> None:
+    new_chk = _checksum_of_file(path, algorithm=entry.alg)
+    if new_chk.lower() != entry.chk.lower():
+        raise ValueError(
+            f"{entry.alg} hash of file '{name}' does not match the known hash: "
+            f"expected {entry.chk}, got {new_chk}."
+        )
+
+
+def _to_file_entries(files: Mapping[str, str | Entry]) -> dict[str, Entry]:
+    return {
+        name: entry if isinstance(entry, Entry) else Entry.from_pooch_string(entry)
+        for name, entry in files.items()
+    }
+
+
+def _to_pooch_registry(files: Mapping[str, str | Entry]) -> dict[str, str]:
+    return {
+        name: f"{entry.alg}:{entry.chk}" if isinstance(entry, Entry) else entry
+        for name, entry in files.items()
+    }
+
+
+# Code taken from Scitacean and Pooch.
+def _checksum_of_file(path: Path, *, algorithm: str) -> str:
+    """Compute the checksum of a local file.
+
+    Parameters
+    ----------
+    path:
+        Path of the file.
+    algorithm:
+        Hash algorithm to use.
+        Can be any algorithm supported by :func:`hashlib.new`.
+
+    Returns
+    -------
+    :
+        The hex digest of the hash.
+    """
+    chk = _new_hash(algorithm)
+    # size based on http://git.savannah.gnu.org/gitweb/?p=coreutils.git;a=blob;f=src/ioblksize.h;h=ed2f4a9c4d77462f357353eb73ee4306c28b37f1;hb=HEAD#l23  # noqa: E501
+    buffer = memoryview(bytearray(128 * 1024))
+    with open(path, "rb", buffering=0) as file:
+        for n in iter(lambda: file.readinto(buffer), 0):
+            chk.update(buffer[:n])
+    return chk.hexdigest()  # type: ignore[no-any-return]
+
+
+def _new_hash(algorithm: str) -> Any:
+    # Try to use a named constructor instead of hashlib.new where possible
+    # because that is supposed to be faster, according to
+    # https://docs.python.org/3/library/hashlib.html#hashlib.new
+    try:
+        return getattr(hashlib, algorithm)()
+    except AttributeError:
+        return hashlib.new(algorithm, usedforsecurity=False)

{essreduce-25.9.0.dist-info → essreduce-25.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: essreduce
-Version: 25.9.0
+Version: 25.10.0
 Summary: Common data reduction tools for the ESS facility
 Author: Scipp contributors
 License-Expression: BSD-3-Clause

{essreduce-25.9.0.dist-info → essreduce-25.10.0.dist-info}/RECORD

@@ -1,5 +1,4 @@
 ess/reduce/__init__.py,sha256=o1pWRP9YGwTukM_k-qlG6KcoXOpMb0PDVH59vod12lw,419
-ess/reduce/data.py,sha256=66FYykg1do4xc_oEEeCu3pyAsyFwniUqEE3ofQIVvwI,5337
 ess/reduce/logging.py,sha256=6n8Czq4LZ3OK9ENlKsWSI1M3KvKv6_HSoUiV4__IUlU,357
 ess/reduce/parameter.py,sha256=4sCfoKOI2HuO_Q7JLH_jAXnEOFANSn5P3NdaOBzhJxc,4635
 ess/reduce/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -7,6 +6,8 @@ ess/reduce/streaming.py,sha256=zbqxQz5dASDq4ZVyx-TdbapBXMyBttImCYz_6WOj4pg,17978
 ess/reduce/ui.py,sha256=zmorAbDwX1cU3ygDT--OP58o0qU7OBcmJz03jPeYSLA,10884
 ess/reduce/uncertainty.py,sha256=LR4O6ApB6Z-W9gC_XW0ajupl8yFG-du0eee1AX_R-gk,6990
 ess/reduce/workflow.py,sha256=738-lcdgsORYfQ4A0UTk2IgnbVxC3jBdpscpaOFIpdc,3114
+ess/reduce/data/__init__.py,sha256=uDtqkmKA_Zwtj6II25zntz9T812XhdCn3tktYev4uyY,486
+ess/reduce/data/_registry.py,sha256=TeEiy2HDKeLXXzSMwvusJqozPA_vPJSz7R8YttgvhxE,11126
 ess/reduce/live/__init__.py,sha256=jPQVhihRVNtEDrE20PoKkclKV2aBF1lS7cCHootgFgI,204
 ess/reduce/live/raw.py,sha256=66qV0G2rP8gK5tXuk-syTlDLE2jT3ehfmSnET7Xzfd0,24392
 ess/reduce/live/roi.py,sha256=Hs-pW98k41WU6Kl3UQ41kQawk80c2QNOQ_WNctLzDPE,3795
@@ -39,9 +40,9 @@ ess/reduce/widgets/_spinner.py,sha256=2VY4Fhfa7HMXox2O7UbofcdKsYG-AJGrsgGJB85nDX
 ess/reduce/widgets/_string_widget.py,sha256=iPAdfANyXHf-nkfhgkyH6gQDklia0LebLTmwi3m-iYQ,1482
 ess/reduce/widgets/_switchable_widget.py,sha256=fjKz99SKLhIF1BLgGVBSKKn3Lu_jYBwDYGeAjbJY3Q8,2390
 ess/reduce/widgets/_vector_widget.py,sha256=aTaBqCFHZQhrIoX6-sSqFWCPePEW8HQt5kUio8jP1t8,1203
-essreduce-25.9.0.dist-info/licenses/LICENSE,sha256=nVEiume4Qj6jMYfSRjHTM2jtJ4FGu0g-5Sdh7osfEYw,1553
-essreduce-25.9.0.dist-info/METADATA,sha256=T-UsR4D9r5xXxCRWnc_LFgI9ogEWKK2GBkfn5MD4cP8,1936
-essreduce-25.9.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-essreduce-25.9.0.dist-info/entry_points.txt,sha256=PMZOIYzCifHMTe4pK3HbhxUwxjFaZizYlLD0td4Isb0,66
-essreduce-25.9.0.dist-info/top_level.txt,sha256=0JxTCgMKPLKtp14wb1-RKisQPQWX7i96innZNvHBr-s,4
-essreduce-25.9.0.dist-info/RECORD,,
+essreduce-25.10.0.dist-info/licenses/LICENSE,sha256=nVEiume4Qj6jMYfSRjHTM2jtJ4FGu0g-5Sdh7osfEYw,1553
+essreduce-25.10.0.dist-info/METADATA,sha256=wn62SjFqeWbNwZMN-K_mBxRJmQy9_rjJx79L6d_UHyk,1937
+essreduce-25.10.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+essreduce-25.10.0.dist-info/entry_points.txt,sha256=PMZOIYzCifHMTe4pK3HbhxUwxjFaZizYlLD0td4Isb0,66
+essreduce-25.10.0.dist-info/top_level.txt,sha256=0JxTCgMKPLKtp14wb1-RKisQPQWX7i96innZNvHBr-s,4
+essreduce-25.10.0.dist-info/RECORD,,

ess/reduce/data.py

@@ -1,154 +0,0 @@
-# SPDX-License-Identifier: BSD-3-Clause
-# Copyright (c) 2025 Scipp contributors (https://github.com/scipp)
-"""Data files bundled with ESSreduce."""
-
-from functools import cache
-from pathlib import Path
-
-
-class Registry:
-    """A registry for data files.
-
-    Note
-    ----
-    This class requires [Pooch](https://www.fatiando.org/pooch/latest/) which
-    is not a hard dependency of ESSreduce and needs to be installed separately.
-    """
-
-    def __init__(
-        self,
-        instrument: str,
-        files: dict[str, str],
-        version: str,
-        retry_if_failed: int = 3,
-    ) -> None:
-        import pooch
-
-        self._registry = pooch.create(
-            path=pooch.os_cache(f'ess/{instrument}'),
-            env=f'ESS_{instrument.upper()}_DATA_DIR',
-            base_url=f'https://public.esss.dk/groups/scipp/ess/{instrument}/'
-            + '{version}/',
-            version=version,
-            registry=files,
-            retry_if_failed=retry_if_failed,
-        )
-        self._unzip_processor = pooch.Unzip()
-
-    def __contains__(self, key: str) -> bool:
-        """Return True if the key is in the registry."""
-        return key in self._registry.registry
-
-    @cache  # noqa: B019
-    def get_path(self, name: str, unzip: bool = False) -> Path:
-        """Get the path to a file in the registry.
-
-        Downloads the file if necessary.
-
-        Note that return values of this method are cached to avoid recomputing
-        potentially expensive checksums.
-        This usually means that the ``Registry`` object itself gets stored until the
-        Python interpreter shuts down.
-        However, registries are small and do not own resources.
-        It is anyway expected that the registry objects are stored at
-        module scope and live until program exit.
-
-        Parameters
-        ----------
-        name:
-            Name of the file to get the path for.
-        unzip:
-            If `True`, unzip the file before returning the path.
-
-        Returns
-        -------
-        :
-            The Path to the file.
-        """
-        return Path(
-            self._registry.fetch(
-                name, processor=self._unzip_processor if unzip else None
-            )
-        )
-
-
-_bifrost_registry = Registry(
-    instrument='bifrost',
-    files={
-        "BIFROST_20240914T053723.h5": "md5:0f2fa5c9a851f8e3a4fa61defaa3752e",
-    },
-    version='1',
-)
-
-
-_dream_registry = Registry(
-    instrument='dream',
-    files={
-        "TEST_977695_00068064.hdf": "md5:9e6ee9ec70d7c5e8c0c93b9e07e8949f",
-    },
-    version='2',
-)
-
-
-_loki_registry = Registry(
-    instrument='loki',
-    files={
-        # Files from LoKI@Larmor detector test experiment
-        #
-        # Background run 1 (no sample, sample holder/can only, no transmission monitor)
-        '60248-2022-02-28_2215.nxs': 'md5:d9f17b95274a0fc6468df7e39df5bf03',
-        # Sample run 1 (sample + sample holder/can, no transmission monitor in beam)
-        '60250-2022-02-28_2215.nxs': 'md5:6a519ceaacbae702a6d08241e86799b1',
-        # Sample run 2 (sample + sample holder/can, no transmission monitor in beam)
-        '60339-2022-02-28_2215.nxs': 'md5:03c86f6389566326bb0cbbd80b8f8c4f',
-        # Background transmission run (sample holder/can + transmission monitor)
-        '60392-2022-02-28_2215.nxs': 'md5:9ecc1a9a2c05a880144afb299fc11042',
-        # Background run 2 (no sample, sample holder/can only, no transmission monitor)
-        '60393-2022-02-28_2215.nxs': 'md5:bf550d0ba29931f11b7450144f658652',
-        # Sample transmission run (sample + sample holder/can + transmission monitor)
-        '60394-2022-02-28_2215.nxs': 'md5:c40f38a62337d86957af925296c4c615',
-        # Analytical model for the I(Q) of the Poly-Gauss sample
-        'PolyGauss_I0-50_Rg-60.h5': 'md5:f5d60d9c2286cb197b8cd4dc82db3d7e',
-        # XML file for the pixel mask
-        'mask_new_July2022.xml': 'md5:421b6dc9db74126ffbc5d88164d017b0',
-    },
-    version='2',
-)
-
-
-def bifrost_simulated_elastic() -> Path:
-    """McStas simulation with elastic incoherent scattering + phonon."""
-    return _bifrost_registry.get_path('BIFROST_20240914T053723.h5')
-
-
-def loki_tutorial_sample_run_60250() -> Path:
-    """Sample run with sample and sample holder/can, no transmission monitor in beam."""
-    return _loki_registry.get_path('60250-2022-02-28_2215.nxs')
-
-
-def loki_tutorial_sample_run_60339() -> Path:
-    """Sample run with sample and sample holder/can, no transmission monitor in beam."""
-    return _loki_registry.get_path('60339-2022-02-28_2215.nxs')
-
-
-def loki_tutorial_background_run_60248() -> Path:
-    """Background run with sample holder/can only, no transmission monitor."""
-    return _loki_registry.get_path('60248-2022-02-28_2215.nxs')
-
-
-def loki_tutorial_background_run_60393() -> Path:
-    """Background run with sample holder/can only, no transmission monitor."""
-    return _loki_registry.get_path('60393-2022-02-28_2215.nxs')
-
-
-def loki_tutorial_sample_transmission_run() -> Path:
-    """Sample transmission run (sample + sample holder/can + transmission monitor)."""
-    return _loki_registry.get_path('60394-2022-02-28_2215.nxs')
-
-
-def dream_coda_test_file() -> Path:
-    """CODA file for DREAM where most pulses have been removed.
-
-    See ``tools/shrink_nexus.py``.
-    """
-    return _dream_registry.get_path('TEST_977695_00068064.hdf')