essreduce 25.11.3__py3-none-any.whl → 25.11.5__py3-none-any.whl

ess/reduce/data/_registry.py

@@ -3,14 +3,14 @@
 
 from __future__ import annotations
 
+import dataclasses
 import hashlib
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
-from dataclasses import dataclass
 from functools import cache
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal
 
 _LOCAL_CACHE_ENV_VAR = "SCIPP_DATA_DIR"
 _LOCAL_REGISTRY_ENV_VAR = "SCIPP_OVERRIDE_DATA_DIR"
@@ -28,8 +28,8 @@ def make_registry(
 
     By default, this function creates a :class:`PoochRegistry` to download files
     via HTTP from an online file store.
-    This can be overridden by setting the environment variable `SCIPP_DATA_DIR` to a
-    path on the local file system.
+    This can be overridden by setting the environment variable
+    ``SCIPP_OVERRIDE_DATA_DIR`` to a path on the local file system.
     In this case, a :class:`LocalRegistry` is returned.
 
     Files are specified as a dict using either the Pooch string format explicitly
@@ -40,7 +40,11 @@ def make_registry(
         ...    "file1.dat": "md5:1234567890abcdef",
         ...    "file2.csv": Entry(alg="md5", chk="abcdef123456789"),
         ...    "folder/nested.dat": "blake2b:1234567890abcdef",
-        ...    "zipped.zip": Entry(alg="blake2b", chk="abcdef123456789", unzip=True),
+        ...    "zipped.zip": Entry(
+        ...        alg="blake2b",
+        ...        chk="abcdef123456789",
+        ...        extractor="unzip"
+        ...    ),
         ... }
 
     In the example above, the specifications for ``file1.dat`` and ``file2.csv`` are
@@ -49,10 +53,11 @@ def make_registry(
     Paths like this must always use forward slashes (/) even on Windows.
 
     As shown above, it is possible to automatically unzip
-    files by specifying ``unzip=True``.
+    files by specifying ``extractor="unzip"``.
     When calling ``registry.get_path("zipped.zip")`` the file will be unzipped and
     a path to the content is returned.
-    This expects that there is only a single file in the zip archive.
+    Similarly, ``extractor="untar"`` specifies that a file needs to be untarred
+    (and possibly un-gzipped).
 
     The complete path to the source file is constructed as follows:
 
@@ -111,7 +116,7 @@ def _check_local_override_path(override: str) -> Path:
     return path
 
 
-@dataclass(frozen=True, slots=True)
+@dataclasses.dataclass(frozen=True, slots=True)
 class Entry:
     """An entry in a registry."""
 
@@ -119,9 +124,18 @@ class Entry:
     """Checksum."""
     alg: str
     """Checksum algorithm."""
-    unzip: bool = False
+    extractor: Literal["unzip", "untar"] | None = None
+    """Processor to extract file contents."""
+
+    unzip: dataclasses.InitVar[bool] = False
     """Whether to unzip the file."""
 
+    def __post_init__(self, unzip: bool) -> None:
+        if self.extractor is not None and unzip:
+            raise TypeError("Set either the 'unzip' argument or 'extractor', not both.")
+        if self.extractor is None and unzip:
+            super().__setattr__("extractor", "unzip")
+
     @classmethod
     def from_pooch_string(cls, pooch_string: str) -> Entry:
         alg, chk = pooch_string.split(":")
@@ -132,7 +146,7 @@ class Registry(ABC):
     def __init__(self, files: Mapping[str, str | Entry]) -> None:
         self._files = _to_file_entries(files)
 
-    @abstractmethod
+    @cache  # noqa: B019
     def get_path(self, name: str) -> Path:
         """Get the path to a file in the registry.
 
@@ -154,9 +168,60 @@ class Registry(ABC):
         :
             The Path to the file.
         """
+        return Path(
+            _expect_single(
+                self._fetch(name, extractor=self._extractor_processor(name)),
+                name,
+            )
+        )
 
-    def _needs_unzip(self, name: str) -> bool:
-        return self._files[name].unzip
+    @cache  # noqa: B019
+    def get_paths(self, name: str) -> list[Path]:
+        """Get the paths to unpacked files from the registry.
+
+        This method downloads the given file, extracts its contents, and returns
+        the paths to all extracted contents.
+        Unlike :meth:`get_path`, this method requires an extractor processor
+        (unzip or untar).
+
+        Depending on the implementation, the file is downloaded if necessary.
+
+        Note that implementations are allowed to cache return values of this method
+        to avoid recomputing potentially expensive checksums.
+        This usually means that the ``Registry`` object itself gets stored until the
+        Python interpreter shuts down.
+        However, registries are small and do not own resources.
+
+        Parameters
+        ----------
+        name:
+            Name of the zipped or tarred file to get the path for.
+
+        Returns
+        -------
+        :
+            The Paths to the files.
+        """
+        if (extractor := self._extractor_processor(name)) is None:
+            raise ValueError(f"File '{name}' is not zipped or tarred.")
+        return [Path(path) for path in self._fetch(name, extractor=extractor)]
+
+    def _extractor_processor_type(self, name: str) -> Any:
+        match self._files[name].extractor:
+            case "unzip":
+                return _pooch_unzip_processor_class()
+            case "untar":
+                return _pooch_untar_processor_class()
+            case None:
+                return None
+
+    @abstractmethod
+    def _extractor_processor(self, name: str) -> Any:
+        """Return an instance of a processor for the given file."""
+
+    @abstractmethod
+    def _fetch(self, name: str, extractor: Any) -> list[str] | str:
+        """Fetch the given file from the registry."""
 
 
 class PoochRegistry(Registry):
@@ -178,24 +243,15 @@ class PoochRegistry(Registry):
         )
         super().__init__(files)
 
-    @cache  # noqa: B019
-    def get_path(self, name: str) -> Path:
-        """Get the path to a file in the registry.
-
-        Downloads the file if necessary.
-        """
-        if self._needs_unzip(name):
-            paths: list[str] = self._registry.fetch(  # type: ignore[assignment]
-                name, processor=self._unzip_processor
-            )
-            return Path(_expect_single_unzipped(paths, name))
-        return Path(self._registry.fetch(name))
+    def _fetch(self, name: str, extractor: Any) -> list[str] | str:
+        return self._registry.fetch(name, processor=extractor)
 
-    @property
-    def _unzip_processor(self) -> Any:
+    def _extractor_processor(self, name: str) -> Any:
         # Create a new processor on demand because reusing the same processor would
         # reuse the same output path for every file.
-        return _import_pooch().Unzip()
+        if (cls := self._extractor_processor_type(name=name)) is not None:
+            return cls()
+        return None
 
 
 class LocalRegistry(Registry):
@@ -217,12 +273,11 @@ class LocalRegistry(Registry):
             base_url=base_url,
             retry_if_failed=retry_if_failed,
         )
-        self._extract_dir = pooch_registry.path
+        self._extract_base_dir = pooch_registry.path
         self._source_path = source_path.resolve().joinpath(*prefix.split("/"), version)
         super().__init__(files)
 
-    @cache  # noqa: B019
-    def get_path(self, name: str) -> Path:
+    def _fetch(self, name: str, extractor: Any) -> list[str] | str:
         """Get the path to a file in the registry."""
         try:
             entry = self._files[name]
@@ -238,24 +293,24 @@ class LocalRegistry(Registry):
 
         _check_hash(name, path, entry)
 
-        if self._needs_unzip(name):
-            return Path(
-                _expect_single_unzipped(
-                    self._unzip_processor(os.fspath(path), "download", None), path
-                )
-            )
-        return path
+        if extractor is not None:
+            return extractor(os.fspath(path), "download", None)
+        return os.fspath(path)
 
     def _local_path(self, name: str) -> Path:
         # Split on "/" because `name` is always a POSIX-style path, but the return
         # value is a system path, i.e., it can be a Windows-style path.
         return self._source_path.joinpath(*name.split("/"))
 
-    @property
-    def _unzip_processor(self) -> Any:
+    def _extract_dir(self, name: str) -> Path:
+        return self._extract_base_dir / name
+
+    def _extractor_processor(self, name: str) -> Any:
         # Create a new processor on demand because reusing the same processor would
         # reuse the same output path for every file.
-        return _import_pooch().Unzip(self._extract_dir)
+        if (cls := self._extractor_processor_type(name=name)) is not None:
+            return cls(extract_dir=self._extract_dir(name))
+        return None
 
 
 def _import_pooch() -> Any:
@@ -288,19 +343,30 @@ def _create_pooch(
     )
 
 
-def _pooch_unzip_processor(extract_dir: Path) -> Any:
+def _pooch_unzip_processor_class() -> Any:
     try:
         import pooch
     except ImportError:
         raise ImportError("You need to install Pooch to unzip files.") from None
 
-    return pooch.processors.Unzip(extract_dir=os.fspath(extract_dir))
+    return pooch.processors.Unzip
+
+
+def _pooch_untar_processor_class() -> Any:
+    try:
+        import pooch
+    except ImportError:
+        raise ImportError("You need to install Pooch to untar files.") from None
+
+    return pooch.processors.Untar
 
 
-def _expect_single_unzipped(paths: list[str], archive: str | os.PathLike) -> str:
+def _expect_single(paths: list[str] | str, archive: str | os.PathLike) -> str:
+    if isinstance(paths, str):
+        return paths
     if len(paths) != 1:
         raise ValueError(
-            f"Expected exactly one file to unzip, got {len(paths)} in "
+            f"Expected exactly one extracted file, got {len(paths)} in "
             f"'{os.fspath(archive)}'."
         )
     return paths[0]

ess/reduce/live/raw.py

@@ -21,7 +21,7 @@ options:
 from __future__ import annotations
 
 from collections.abc import Callable, Sequence
-from math import ceil
+from math import ceil, prod
 from typing import Literal, NewType
 
 import numpy as np
@@ -138,6 +138,139 @@ class Histogrammer:
         return self._hist(replicated, coords=self._coords) / self.replicas
 
 
+class LogicalView:
+    """
+    Logical view for detector data.
+
+    Implements a view by applying a user-defined transform (e.g., fold or slice
+    operations) optionally followed by reduction (summing) over specified dimensions.
+    Transformation and reduction must be specified separately for `LogicalView` to
+    construct a mapping from output indices to input indices. So `transform` must not
+    perform any reductions.
+
+    This class provides both data transformation (__call__) and index mapping
+    (input_indices) using the same transform, ensuring consistency for ROI filtering.
+    """
+
+    def __init__(
+        self,
+        transform: Callable[[sc.DataArray], sc.DataArray],
+        reduction_dim: str | list[str] | None = None,
+        input_sizes: dict[str, int] | None = None,
+    ):
+        """
+        Create a logical view.
+
+        Parameters
+        ----------
+        transform:
+            Callable that transforms input data by reshaping or selecting pixels.
+            Examples:
+            - Fold: lambda da: da.fold('x', {'x': 512, 'x_bin': 2})
+            - Slice: lambda da: da['z', 0]  (select front layer of volume)
+            - Combined: lambda da: da.fold('x', {'x': 4, 'z': 8})['z', 0]
+        reduction_dim:
+            Dimension(s) to sum over after applying transform. If None or empty,
+            no reduction is performed (pure transform).
+            Example: 'x_bin' or ['x_bin', 'y_bin']
+        input_sizes:
+            Dictionary defining the input dimension sizes.
+            Required for input_indices().
+            If not provided, input_indices() will raise an error.
+            When used with RollingDetectorView, this is automatically
+            inferred from detector_number.
+        """
+        self._transform = transform
+        if reduction_dim is None:
+            self._reduction_dim = []
+        elif isinstance(reduction_dim, str):
+            self._reduction_dim = [reduction_dim]
+        else:
+            self._reduction_dim = reduction_dim
+        self._input_sizes = input_sizes
+
+    @property
+    def replicas(self) -> int:
+        """Number of replicas. Always 1 for LogicalView."""
+        return 1
+
+    def __call__(self, da: sc.DataArray) -> sc.DataArray:
+        """
+        Apply transform and optionally sum over reduction dimensions.
+
+        Parameters
+        ----------
+        da:
+            Data to downsample.
+
+        Returns
+        -------
+        :
+            Transformed (and optionally reduced) data array.
+        """
+        transformed = self._transform(da)
+        if self._reduction_dim:
+            return transformed.sum(self._reduction_dim)
+        return transformed
+
+    def input_indices(self) -> sc.DataArray:
+        """
+        Create index mapping for ROI filtering.
+
+        Returns a DataArray mapping output pixels to input indices (as indices into
+        the flattened input array). If reduction dimensions are specified, returns
+        binned data where each output pixel contains a list of contributing input
+        indices. If no reduction, returns dense indices (1:1 mapping).
+
+        Returns
+        -------
+        :
+            DataArray mapping output pixels to input indices.
+
+        Raises
+        ------
+        ValueError:
+            If input_sizes was not provided during initialization.
+        """
+        if self._input_sizes is None:
+            raise ValueError(
+                "input_sizes is required for input_indices(). "
+                "Provide it during LogicalView initialization."
+            )
+
+        # Create sequential indices (0, 1, 2, ...) and fold to input shape
+        total_size = prod(self._input_sizes.values())
+        indices = sc.arange('_temp', total_size, dtype='int64', unit=None)
+        indices = indices.fold(dim='_temp', sizes=self._input_sizes)
+
+        # Apply transform to get the grouping structure
+        transformed = self._transform(sc.DataArray(data=indices))
+
+        if not self._reduction_dim:
+            # No reduction: 1:1 mapping, return dense indices
+            return sc.DataArray(data=transformed.data)
+
+        # Flatten reduction dimensions to a single dimension.
+        # First transpose to make reduction dims contiguous at the end.
+        output_dims = [d for d in transformed.dims if d not in self._reduction_dim]
+        transformed = transformed.transpose(output_dims + self._reduction_dim)
+        transformed = transformed.flatten(dims=self._reduction_dim, to='_reduction')
+
+        # Convert dense array to binned structure where each output pixel
+        # contains a bin with the indices of contributing input pixels.
+        bin_size = transformed.sizes['_reduction']
+        output_shape = [transformed.sizes[d] for d in output_dims]
+        data_flat = transformed.data.flatten(to='_flat')
+        begin = sc.array(
+            dims=output_dims,
+            values=np.arange(0, data_flat.size, bin_size, dtype=np.int64).reshape(
+                output_shape
+            ),
+            unit=None,
+        )
+        return sc.DataArray(sc.bins(begin=begin, dim='_flat', data=data_flat))
+
+
 class Detector:
     def __init__(self, detector_number: sc.Variable):
         self._data = sc.DataArray(
@@ -220,6 +353,48 @@ class RollingDetectorView(Detector):
         self._cumulative: sc.DataArray
         self.clear_counts()
 
+    @staticmethod
+    def with_logical_view(
+        *,
+        detector_number: sc.Variable,
+        window: int,
+        transform: Callable[[sc.DataArray], sc.DataArray],
+        reduction_dim: str | list[str] | None = None,
+    ) -> RollingDetectorView:
+        """
+        Create a RollingDetectorView with a LogicalView projection.
+
+        This factory method creates a LogicalView with input_sizes
+        automatically inferred from detector_number.sizes.
+
+        Parameters
+        ----------
+        detector_number:
+            Detector number for each pixel.
+        window:
+            Size of the rolling window.
+        transform:
+            Transform function for the LogicalView.
+        reduction_dim:
+            Reduction dimension(s) for the LogicalView. If None or empty,
+            no reduction is performed (pure transform).
+
+        Returns
+        -------
+        :
+            RollingDetectorView with LogicalView projection.
+        """
+        view = LogicalView(
+            transform=transform,
+            reduction_dim=reduction_dim,
+            input_sizes=dict(detector_number.sizes),
+        )
+        return RollingDetectorView(
+            detector_number=detector_number,
+            window=window,
+            projection=view,
+        )
+
     @property
     def max_window(self) -> int:
         return self._window
@@ -249,9 +424,11 @@ class RollingDetectorView(Detector):
     def make_roi_filter(self) -> roi.ROIFilter:
         """Return a ROI filter operating via the projection plane of the view."""
         norm = 1.0
-        if isinstance(self._projection, Histogrammer):
+        # Use duck typing: check if projection has input_indices method
+        if hasattr(self._projection, 'input_indices'):
             indices = self._projection.input_indices()
-            norm = self._projection.replicas
+            # Get replicas property if it exists (Histogrammer has it, default to 1.0)
+            norm = getattr(self._projection, 'replicas', 1.0)
         else:
             indices = sc.ones(sizes=self.data.sizes, dtype='int32', unit=None)
             indices = sc.cumsum(indices, mode='exclusive')
@@ -292,10 +469,11 @@ class RollingDetectorView(Detector):
                     if not sc.identical(det_num, self.detector_number):
                         raise sc.CoordError("Mismatching detector numbers in weights.")
                 weights = weights.data
-        if isinstance(self._projection, Histogrammer):
+        # Use duck typing: check for apply_full method (Histogrammer)
+        if hasattr(self._projection, 'apply_full'):
             xs = self._projection.apply_full(weights)  # Use all replicas
         elif self._projection is not None:
-            xs = self._projection(weights)
+            xs = self._projection(weights)  # LogicalDownsampler or callable
         else:
             xs = weights.copy()
         nonempty = xs.values[xs.values > 0]

{essreduce-25.11.3.dist-info → essreduce-25.11.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: essreduce
-Version: 25.11.3
+Version: 25.11.5
 Summary: Common data reduction tools for the ESS facility
 Author: Scipp contributors
 License-Expression: BSD-3-Clause

{essreduce-25.11.3.dist-info → essreduce-25.11.5.dist-info}/RECORD

@@ -8,9 +8,9 @@ ess/reduce/ui.py,sha256=zmorAbDwX1cU3ygDT--OP58o0qU7OBcmJz03jPeYSLA,10884
 ess/reduce/uncertainty.py,sha256=LR4O6ApB6Z-W9gC_XW0ajupl8yFG-du0eee1AX_R-gk,6990
 ess/reduce/workflow.py,sha256=738-lcdgsORYfQ4A0UTk2IgnbVxC3jBdpscpaOFIpdc,3114
 ess/reduce/data/__init__.py,sha256=uDtqkmKA_Zwtj6II25zntz9T812XhdCn3tktYev4uyY,486
-ess/reduce/data/_registry.py,sha256=50qY36y5gd2i3JP0Ks6bXApGcW6l70qA6riO0m9Bz4o,11475
+ess/reduce/data/_registry.py,sha256=dJ4DymZsknFi3F6kHZmPTkEJ774tRXsZgD6R6v6sz0o,13987
 ess/reduce/live/__init__.py,sha256=jPQVhihRVNtEDrE20PoKkclKV2aBF1lS7cCHootgFgI,204
-ess/reduce/live/raw.py,sha256=d86s5fBjdf6tI_FWNMG4ZG67GCY2JADOobrinncNIjE,24367
+ess/reduce/live/raw.py,sha256=CkPqp4VMNvj0IcFPp1J0n7sVt5PNKdIXnDlALCg9W_Q,31031
 ess/reduce/live/roi.py,sha256=Hs-pW98k41WU6Kl3UQ41kQawk80c2QNOQ_WNctLzDPE,3795
 ess/reduce/live/workflow.py,sha256=bsbwvTqPhRO6mC__3b7MgU7DWwAnOvGvG-t2n22EKq8,4285
 ess/reduce/nexus/__init__.py,sha256=xXc982vZqRba4jR4z5hA2iim17Z7niw4KlS1aLFbn1Q,1107
@@ -41,9 +41,9 @@ ess/reduce/widgets/_spinner.py,sha256=2VY4Fhfa7HMXox2O7UbofcdKsYG-AJGrsgGJB85nDX
 ess/reduce/widgets/_string_widget.py,sha256=iPAdfANyXHf-nkfhgkyH6gQDklia0LebLTmwi3m-iYQ,1482
 ess/reduce/widgets/_switchable_widget.py,sha256=fjKz99SKLhIF1BLgGVBSKKn3Lu_jYBwDYGeAjbJY3Q8,2390
 ess/reduce/widgets/_vector_widget.py,sha256=aTaBqCFHZQhrIoX6-sSqFWCPePEW8HQt5kUio8jP1t8,1203
-essreduce-25.11.3.dist-info/licenses/LICENSE,sha256=nVEiume4Qj6jMYfSRjHTM2jtJ4FGu0g-5Sdh7osfEYw,1553
-essreduce-25.11.3.dist-info/METADATA,sha256=RG8gLrZiQjprpTwz70dd-300cLF2jZeOEIigmwWFBOY,1937
-essreduce-25.11.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-essreduce-25.11.3.dist-info/entry_points.txt,sha256=PMZOIYzCifHMTe4pK3HbhxUwxjFaZizYlLD0td4Isb0,66
-essreduce-25.11.3.dist-info/top_level.txt,sha256=0JxTCgMKPLKtp14wb1-RKisQPQWX7i96innZNvHBr-s,4
-essreduce-25.11.3.dist-info/RECORD,,
+essreduce-25.11.5.dist-info/licenses/LICENSE,sha256=nVEiume4Qj6jMYfSRjHTM2jtJ4FGu0g-5Sdh7osfEYw,1553
+essreduce-25.11.5.dist-info/METADATA,sha256=4FcweJeK7mrl5NtA1cvbTWyem_tw1I6XA_TFNQwL-qc,1937
+essreduce-25.11.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+essreduce-25.11.5.dist-info/entry_points.txt,sha256=PMZOIYzCifHMTe4pK3HbhxUwxjFaZizYlLD0td4Isb0,66
+essreduce-25.11.5.dist-info/top_level.txt,sha256=0JxTCgMKPLKtp14wb1-RKisQPQWX7i96innZNvHBr-s,4
+essreduce-25.11.5.dist-info/RECORD,,