essreduce 25.2.6__tar.gz → 25.3.1__tar.gz

{essreduce-25.2.6/src/essreduce.egg-info → essreduce-25.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: essreduce
-Version: 25.2.6
+Version: 25.3.1
 Summary: Common data reduction tools for the ESS facility
 Author: Scipp contributors
 License: BSD 3-Clause License

{essreduce-25.2.6 → essreduce-25.3.1}/src/ess/reduce/nexus/_nexus_loader.py

@@ -85,7 +85,7 @@ def load_metadata(
     entry_name: NeXusEntryName | None = None,
     definitions: Mapping | NoNewDefinitionsType = NoNewDefinitions,
 ) -> _Model:
-    with _open_nexus_file(file_path, definitions=definitions) as f:
+    with open_nexus_file(file_path, definitions=definitions) as f:
         entry = _unique_child_group(f, snx.NXentry, entry_name)
         return model.from_nexus_entry(entry)
 
@@ -113,7 +113,7 @@ def compute_component_position(dg: sc.DataGroup) -> sc.DataGroup:
     )
 
 
-def _open_nexus_file(
+def open_nexus_file(
     file_path: FilePath | NeXusFile | NeXusGroup,
     definitions: Mapping | None | NoNewDefinitionsType = NoNewDefinitions,
     *,
@@ -212,7 +212,7 @@ def _open_component_parent(
     """Locate the parent group of a NeXus component."""
     file_path = location.filename
     entry_name = location.entry_name
-    with _open_nexus_file(file_path, definitions=definitions) as f:
+    with open_nexus_file(file_path, definitions=definitions) as f:
         entry = _unique_child_group(f, snx.NXentry, entry_name)
         if nx_class is snx.NXsample:
             yield entry
@@ -357,7 +357,7 @@ def load_data(
     :
         Data array with events or a histogram.
     """
-    with _open_nexus_file(file_path, definitions=definitions) as f:
+    with open_nexus_file(file_path, definitions=definitions) as f:
         entry = _unique_child_group(f, snx.NXentry, entry_name)
         instrument = _unique_child_group(entry, snx.NXinstrument, None)
         component = instrument[component_name]
@@ -554,7 +554,7 @@ def _parse_monitor(group: snx.Group) -> NeXusMonitorInfo:
 
 def read_nexus_file_info(file_path: FilePath | NeXusFile | NeXusGroup) -> NeXusFileInfo:
     """Opens and inspects a NeXus file, returning a summary of its contents."""
-    with _open_nexus_file(file_path) as f:
+    with open_nexus_file(file_path) as f:
         entry = _unique_child_group(f, snx.NXentry, None)
         instrument = _unique_child_group(entry, snx.NXinstrument, None)
         detectors = {}

{essreduce-25.2.6 → essreduce-25.3.1}/src/ess/reduce/nexus/workflow.py

@@ -62,7 +62,9 @@ def file_path_to_file_spec(
     filename: Filename[RunType], preopen: PreopenNeXusFile
 ) -> NeXusFileSpec[RunType]:
     return NeXusFileSpec[RunType](
-        snx.File(filename, definitions=definitions) if preopen else filename
+        nexus.open_nexus_file(filename, definitions=definitions)
+        if preopen
+        else filename
     )
 
 

{essreduce-25.2.6 → essreduce-25.3.1}/src/ess/reduce/streaming.py

@@ -240,6 +240,22 @@ class StreamProcessor:
     processing based on the input keys. In particular, it is the responsibility of the
     user to ensure that the workflow is "linear" with respect to the dynamic keys up to
     the accumulation keys.
+
+    Similarly, the stream processor cannot determine from the workflow structure whether
+    context updates are compatible with the accumulated data. Accumulators are not
+    cleared automatically. This is best illustrated with an example:
+
+    - If the context is the detector rotation angle, and we accumulate I(Q) (or a
+      prerequisite of I(Q)), then updating the detector angle context is compatible with
+      previous data, assuming Q for each new chunk is computed based on the angle.
+    - If the context is the sample temperature, and we accumulate I(Q), then updating
+      the temperature context is not compatible with previous data. Accumulating I(Q, T)
+      could be compatible in this case.
+
+    Since the correctness cannot be determined from the workflow structure, we recommend
+    implementing processing steps in a way to catch such problems. For example, adding
+    the temperature as a coordinate to the I(Q) data array should allow for
+    automatically raising in the accumulator if the temperature changes.
     """
 
     def __init__(
@@ -247,6 +263,7 @@ class StreamProcessor:
         base_workflow: sciline.Pipeline,
         *,
         dynamic_keys: tuple[sciline.typing.Key, ...],
+        context_keys: tuple[sciline.typing.Key, ...] = (),
         target_keys: tuple[sciline.typing.Key, ...],
         accumulators: dict[sciline.typing.Key, Accumulator | Callable[..., Accumulator]]
         | tuple[sciline.typing.Key, ...],
@@ -260,7 +277,12 @@ class StreamProcessor:
         base_workflow:
             Workflow to be used for processing chunks.
         dynamic_keys:
-            Keys that are expected to be updated with each chunk.
+            Keys that are expected to be updated with each chunk. These keys cannot
+            depend on each other or on context_keys.
+        context_keys:
+            Keys that define context for processing chunks and may change occasionally.
+            These keys cannot overlap with dynamic_keys or depend on each other or on
+            dynamic_keys.
         target_keys:
             Keys to be computed and returned.
         accumulators:
@@ -275,21 +297,59 @@ class StreamProcessor:
             unless the values for these keys are valid for all chunks comprised in the
             final accumulators at the point where :py:meth:`finalize` is called.
         """
+        self._dynamic_keys = set(dynamic_keys)
+        self._context_keys = set(context_keys)
+
+        # Validate that dynamic and context keys do not overlap
+        overlap = self._dynamic_keys & self._context_keys
+        if overlap:
+            raise ValueError(f"Keys cannot be both dynamic and context: {overlap}")
+
+        # Check dynamic/context keys don't depend on other dynamic/context keys
+        graph = base_workflow.underlying_graph
+        special_keys = self._dynamic_keys | self._context_keys
+        for key in special_keys:
+            if key not in graph:
+                continue
+            ancestors = nx.ancestors(graph, key)
+            special_ancestors = ancestors & special_keys
+            downstream = 'Dynamic' if key in self._dynamic_keys else 'Context'
+            if special_ancestors:
+                raise ValueError(
+                    f"{downstream} key '{key}' depends on other dynamic/context keys: "
+                    f"{special_ancestors}. This is not supported."
+                )
+
         workflow = sciline.Pipeline()
         for key in target_keys:
             workflow[key] = base_workflow[key]
         for key in dynamic_keys:
             workflow[key] = None  # hack to prune branches
-
-        self._dynamic_keys = set(dynamic_keys)
+        for key in context_keys:
+            workflow[key] = None
 
         # Find and pre-compute static nodes as far down the graph as possible
-        # See also https://github.com/scipp/sciline/issues/148.
-        nodes = _find_descendants(workflow, dynamic_keys)
-        parents = _find_parents(workflow, nodes) - nodes
-        for key, value in base_workflow.compute(parents).items():
+        nodes = _find_descendants(workflow, dynamic_keys + context_keys)
+        last_static = _find_parents(workflow, nodes) - nodes
+        for key, value in base_workflow.compute(last_static).items():
             workflow[key] = value
 
+        # Nodes that may need updating on context change but should be cached otherwise.
+        dynamic_nodes = _find_descendants(workflow, dynamic_keys)
+        # Nodes as far "down" in the graph as possible, right before the dynamic nodes.
+        # This also includes target keys that are not dynamic but context-dependent.
+        context_to_cache = (
+            (_find_parents(workflow, dynamic_nodes) | set(target_keys)) - dynamic_nodes
+        ) & _find_descendants(workflow, context_keys)
+        graph = workflow.underlying_graph
+        self._context_key_to_cached_context_nodes_map = {
+            context_key: ({context_key} | nx.descendants(graph, context_key))
+            & context_to_cache
+            for context_key in self._context_keys
+            if context_key in graph
+        }
+
+        self._context_workflow = workflow.copy()
         self._process_chunk_workflow = workflow.copy()
         self._finalize_workflow = workflow.copy()
         self._accumulators = (
@@ -299,7 +359,6 @@ class StreamProcessor:
         )
 
         # Map each accumulator to its dependent dynamic keys
-        graph = workflow.underlying_graph
         self._accumulator_dependencies = {
             acc_key: nx.ancestors(graph, acc_key) & self._dynamic_keys
             for acc_key in self._accumulators
@@ -323,6 +382,30 @@ class StreamProcessor:
         self._target_keys = target_keys
         self._allow_bypass = allow_bypass
 
+    def set_context(self, context: dict[sciline.typing.Key, Any]) -> None:
+        """
+        Set the context for processing chunks.
+
+        Parameters
+        ----------
+        context:
+            Context to be set.
+        """
+        needs_recompute = set()
+        for key in context:
+            if key not in self._context_keys:
+                raise ValueError(f"Key '{key}' is not a context key")
+            needs_recompute |= self._context_key_to_cached_context_nodes_map[key]
+        for key, value in context.items():
+            self._context_workflow[key] = value
+        results = self._context_workflow.compute(needs_recompute)
+        for key, value in results.items():
+            if key in self._target_keys:
+                # Context-dependent key is direct target, independent of dynamic nodes.
+                self._finalize_workflow[key] = value
+            else:
+                self._process_chunk_workflow[key] = value
+
     def add_chunk(
         self, chunks: dict[sciline.typing.Key, Any]
     ) -> dict[sciline.typing.Key, Any]:

{essreduce-25.2.6 → essreduce-25.3.1}/src/ess/reduce/time_of_flight/__init__.py

@@ -6,9 +6,9 @@ Utilities for computing real neutron time-of-flight from chopper settings and
 neutron time-of-arrival at the detectors.
 """
 
+from .eto_to_tof import default_parameters, providers, resample_tof_data
 from .simulation import simulate_beamline
 from .to_events import to_events
-from .toa_to_tof import default_parameters, providers, resample_tof_data
 from .types import (
     DistanceResolution,
     LookupTableRelativeErrorThreshold,

essreduce-25.2.6/src/ess/reduce/time_of_flight/toa_to_tof.py → essreduce-25.3.1/src/ess/reduce/time_of_flight/eto_to_tof.py

@@ -284,38 +284,90 @@ def compute_tof_lookup_table(
     )
 
 
-def _make_tof_interpolator(
-    lookup: sc.DataArray, distance_unit: str, time_unit: str
-) -> Callable:
-    from scipy.interpolate import RegularGridInterpolator
-
-    # TODO: to make use of multi-threading, we could write our own interpolator.
-    # This should be simple enough as we are making the bins linspace, so computing
-    # bin indices is fast.
-
-    # In the pulse dimension, it could be that for a given event_time_offset and
-    # distance, a tof value is finite in one pulse and NaN in the other.
-    # When using the bilinear interpolation, even if the value of the requested point is
-    # exactly 0 or 1 (in the case of pulse_stride=2), the interpolator will still
-    # use all 4 corners surrounding the point. This means that if one of the corners
-    # is NaN, the result will be NaN.
-    # Here, we use a trick where we duplicate the lookup values in the 'pulse' dimension
-    # so that the interpolator has values on bin edges for that dimension.
-    # The interpolator raises an error if axes coordinates are not strictly monotonic,
-    # so we cannot use e.g. [-0.5, 0.5, 0.5, 1.5] in the case of pulse_stride=2.
-    # Instead we use [-0.25, 0.25, 0.75, 1.25].
-    base_grid = np.arange(float(lookup.sizes["pulse"]))
-    return RegularGridInterpolator(
-        (
-            np.sort(np.concatenate([base_grid - 0.25, base_grid + 0.25])),
-            lookup.coords["distance"].to(unit=distance_unit, copy=False).values,
-            lookup.coords["event_time_offset"].to(unit=time_unit, copy=False).values,
-        ),
-        np.repeat(lookup.data.to(unit=time_unit, copy=False).values, 2, axis=0),
-        method="linear",
-        bounds_error=False,
-        fill_value=np.nan,
-    )
+class TofInterpolator:
+    def __init__(self, lookup: sc.DataArray, distance_unit: str, time_unit: str):
+        from scipy.interpolate import RegularGridInterpolator
+
+        # TODO: to make use of multi-threading, we could write our own interpolator.
+        # This should be simple enough as we are making the bins linspace, so computing
+        # bin indices is fast.
+
+        self._distance_unit = distance_unit
+        self._time_unit = time_unit
+
+        # In the pulse dimension, it could be that for a given event_time_offset and
+        # distance, a tof value is finite in one pulse and NaN in the other.
+        # When using the bilinear interpolation, even if the value of the requested
+        # point is exactly 0 or 1 (in the case of pulse_stride=2), the interpolator
+        # will still use all 4 corners surrounding the point. This means that if one of
+        # the corners is NaN, the result will be NaN.
+        # Here, we use a trick where we duplicate the lookup values in the 'pulse'
+        # dimension so that the interpolator has values on bin edges for that dimension.
+        # The interpolator raises an error if axes coordinates are not strictly
+        # monotonic, so we cannot use e.g. [-0.5, 0.5, 0.5, 1.5] in the case of
+        # pulse_stride=2. Instead we use [-0.25, 0.25, 0.75, 1.25].
+        base_grid = np.arange(float(lookup.sizes["pulse"]))
+        self._interpolator = RegularGridInterpolator(
+            (
+                np.sort(np.concatenate([base_grid - 0.25, base_grid + 0.25])),
+                lookup.coords["distance"].to(unit=distance_unit, copy=False).values,
+                lookup.coords["event_time_offset"]
+                .to(unit=self._time_unit, copy=False)
+                .values,
+            ),
+            np.repeat(
+                lookup.data.to(unit=self._time_unit, copy=False).values, 2, axis=0
+            ),
+            method="linear",
+            bounds_error=False,
+            fill_value=np.nan,
+        )
+
+    def __call__(
+        self,
+        pulse_index: sc.Variable,
+        ltotal: sc.Variable,
+        event_time_offset: sc.Variable,
+    ) -> sc.Variable:
+        if pulse_index.unit not in ("", None):
+            raise sc.UnitError(
+                "pulse_index must have unit dimensionless or None, "
+                f"but got unit: {pulse_index.unit}."
+            )
+        if ltotal.unit != self._distance_unit:
+            raise sc.UnitError(
+                f"ltotal must have unit: {self._distance_unit}, "
+                f"but got unit: {ltotal.unit}."
+            )
+        if event_time_offset.unit != self._time_unit:
+            raise sc.UnitError(
+                f"event_time_offset must have unit: {self._time_unit}, "
+                f"but got unit: {event_time_offset.unit}."
+            )
+        out_dims = event_time_offset.dims
+        pulse_index = pulse_index.values
+        ltotal = ltotal.values
+        event_time_offset = event_time_offset.values
+        # Check bounds for pulse_index and ltotal.
+        # We do not check the event_time_offset dimension because histogrammed monitors
+        # often have binning which can be anything (does not necessarily stop at 71ms).
+        # Raising an error here would be too restrictive, and warnings would add noise
+        # to the workflows.
+        for i, (name, values) in enumerate(
+            {'pulse_index': pulse_index, 'ltotal': ltotal}.items()
+        ):
+            vmin = self._interpolator.grid[i][0]
+            vmax = self._interpolator.grid[i][-1]
+            if np.any(values < vmin) or np.any(values > vmax):
+                raise ValueError(
+                    "Some requested values are outside of lookup table bounds for "
+                    f"axis {i}: {name}, min: {vmin}, max: {vmax}."
+                )
+        return sc.array(
+            dims=out_dims,
+            values=self._interpolator((pulse_index, ltotal, event_time_offset)),
+            unit=self._time_unit,
+        )
 
 
 def _time_of_flight_data_histogram(
@@ -360,17 +412,11 @@ def _time_of_flight_data_histogram(
     )
     pulse_index = sc.zeros(sizes=etos.sizes)
 
-    # Create 2D interpolator
-    interp = _make_tof_interpolator(
-        lookup, distance_unit=ltotal.unit, time_unit=eto_unit
-    )
+    # Create linear interpolator
+    interp = TofInterpolator(lookup, distance_unit=ltotal.unit, time_unit=eto_unit)
 
     # Compute time-of-flight of the bin edges using the interpolator
-    tofs = sc.array(
-        dims=etos.dims,
-        values=interp((pulse_index.values, ltotal.values, etos.values)),
-        unit=eto_unit,
-    )
+    tofs = interp(pulse_index=pulse_index, ltotal=ltotal, event_time_offset=etos)
 
     return rebinned.assign_coords(tof=tofs)
 
@@ -380,7 +426,7 @@ def _guess_pulse_stride_offset(
     ltotal: sc.Variable,
     event_time_offset: sc.Variable,
     pulse_stride: int,
-    interp: Callable,
+    interp: TofInterpolator,
 ) -> int:
     """
     Using the minimum ``event_time_zero`` to calculate a reference time when computing
@@ -408,21 +454,29 @@ def _guess_pulse_stride_offset(
     pulse_stride:
         Stride of used pulses.
     interp:
-        2D interpolator for the lookup table.
+        Interpolator for the lookup table.
     """
     tofs = {}
     # Choose a few random events to compute the time-of-flight
     inds = np.random.choice(
         len(event_time_offset), min(5000, len(event_time_offset)), replace=False
     )
-    pulse_index_values = pulse_index.values[inds]
-    ltotal_values = ltotal.values[inds]
-    etos_values = event_time_offset.values[inds]
+    pulse_index = sc.array(
+        dims=pulse_index.dims,
+        values=pulse_index.values[inds],
+        unit=pulse_index.unit,
+    )
+    ltotal = sc.array(dims=ltotal.dims, values=ltotal.values[inds], unit=ltotal.unit)
+    etos = sc.array(
+        dims=event_time_offset.dims,
+        values=event_time_offset.values[inds],
+        unit=event_time_offset.unit,
+    )
     for i in range(pulse_stride):
-        pulse_inds = (pulse_index_values + i) % pulse_stride
-        tofs[i] = interp((pulse_inds, ltotal_values, etos_values))
+        pulse_inds = (pulse_index + i) % pulse_stride
+        tofs[i] = interp(pulse_index=pulse_inds, ltotal=ltotal, event_time_offset=etos)
     # Find the entry in the list with the least number of nan values
-    return sorted(tofs, key=lambda x: np.isnan(tofs[x]).sum())[0]
+    return sorted(tofs, key=lambda x: sc.isnan(tofs[x]).sum())[0]
 
 
 def _time_of_flight_data_events(
@@ -436,10 +490,8 @@ def _time_of_flight_data_events(
     etos = da.bins.coords["event_time_offset"]
     eto_unit = elem_unit(etos)
 
-    # Create 2D interpolator
-    interp = _make_tof_interpolator(
-        lookup, distance_unit=ltotal.unit, time_unit=eto_unit
-    )
+    # Create linear interpolator
+    interp = TofInterpolator(lookup, distance_unit=ltotal.unit, time_unit=eto_unit)
 
     # Operate on events (broadcast distances to all events)
     ltotal = sc.bins_like(etos, ltotal).bins.constituents["data"]
@@ -491,11 +543,7 @@ def _time_of_flight_data_events(
         pulse_index %= pulse_stride
 
     # Compute time-of-flight for all neutrons using the interpolator
-    tofs = sc.array(
-        dims=etos.dims,
-        values=interp((pulse_index.values, ltotal.values, etos.values)),
-        unit=eto_unit,
-    )
+    tofs = interp(pulse_index=pulse_index, ltotal=ltotal, event_time_offset=etos)
 
     parts = da.bins.constituents
     parts["data"] = tofs

{essreduce-25.2.6 → essreduce-25.3.1/src/essreduce.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: essreduce
-Version: 25.2.6
+Version: 25.3.1
 Summary: Common data reduction tools for the ESS facility
 Author: Scipp contributors
 License: BSD 3-Clause License

{essreduce-25.2.6 → essreduce-25.3.1}/src/essreduce.egg-info/SOURCES.txt

@@ -89,10 +89,10 @@ src/ess/reduce/nexus/types.py
 src/ess/reduce/nexus/workflow.py
 src/ess/reduce/scripts/grow_nexus.py
 src/ess/reduce/time_of_flight/__init__.py
+src/ess/reduce/time_of_flight/eto_to_tof.py
 src/ess/reduce/time_of_flight/fakes.py
 src/ess/reduce/time_of_flight/simulation.py
 src/ess/reduce/time_of_flight/to_events.py
-src/ess/reduce/time_of_flight/toa_to_tof.py
 src/ess/reduce/time_of_flight/types.py
 src/ess/reduce/widgets/__init__.py
 src/ess/reduce/widgets/_base.py

{essreduce-25.2.6 → essreduce-25.3.1}/tests/nexus/nexus_loader_test.py

@@ -633,7 +633,7 @@ def compute_component_position_returns_input_if_no_depends_on() -> None:
 # h5py cannot open files on these systems with file locks.
 # We cannot reasonably emulate this within Python tests.
 # So the following tests only check the behaviour on a basic level.
-# The tests use the private `_open_nexus_file` directly to focus on what matters.
+# The tests use the private `open_nexus_file` directly to focus on what matters.
 #
 # A file may already be open in this or another process.
 # We should still be able to open it
@@ -652,13 +652,13 @@ def compute_component_position_returns_input_if_no_depends_on() -> None:
     ],
 )
 def test_open_nexus_file_multiple_times(tmp_path: Path, locks: tuple[Any, Any]) -> None:
-    from ess.reduce.nexus._nexus_loader import _open_nexus_file
+    from ess.reduce.nexus._nexus_loader import open_nexus_file
 
     path = FilePath(tmp_path / "file.nxs")
     with snx.File(path, "w"):
         pass
-    with _open_nexus_file(path, locking=locks[0]) as f1:
-        with _open_nexus_file(path, locking=locks[1]) as f2:
+    with open_nexus_file(path, locking=locks[0]) as f1:
+        with open_nexus_file(path, locking=locks[1]) as f2:
             assert f1.name == f2.name
 
 
@@ -669,15 +669,15 @@ def _in_conda_env():
 def _test_open_nexus_file_with_mismatched_locking(
     tmp_path: Path, locks: tuple[Any, Any]
 ) -> None:
-    from ess.reduce.nexus._nexus_loader import _open_nexus_file
+    from ess.reduce.nexus._nexus_loader import open_nexus_file
 
     path = FilePath(tmp_path / "file.nxs")
     with snx.File(path, "w"):
         pass
 
-    with _open_nexus_file(path, locking=locks[0]):
+    with open_nexus_file(path, locking=locks[0]):
         with pytest.raises(OSError, match="flag values don't match"):
-            _ = _open_nexus_file(path, locking=locks[1])
+            _ = open_nexus_file(path, locking=locks[1])
 
 
 @pytest.mark.skipif(
@@ -721,7 +721,7 @@ def test_open_nexus_file_with_mismatched_locking_all(
 
 
 def test_open_nonexisting_file_raises_filenotfounderror():
-    from ess.reduce.nexus._nexus_loader import _open_nexus_file
+    from ess.reduce.nexus._nexus_loader import open_nexus_file
 
     with pytest.raises(FileNotFoundError):
-        _open_nexus_file(nexus.types.FilePath(Path("doesnotexist.hdf")))
+        open_nexus_file(nexus.types.FilePath(Path("doesnotexist.hdf")))

{essreduce-25.2.6 → essreduce-25.3.1}/tests/nexus/workflow_test.py

@@ -26,6 +26,7 @@ from ess.reduce.nexus.types import (
     NeXusComponentLocationSpec,
     NeXusName,
     NeXusTransformation,
+    PreopenNeXusFile,
     RunType,
     SampleRun,
     TimeInterval,
@@ -533,11 +534,13 @@ def test_load_detector_workflow() -> None:
     assert da.dims == ('detector_number',)
 
 
-def test_generic_nexus_workflow() -> None:
+@pytest.mark.parametrize('preopen', [True, False])
+def test_generic_nexus_workflow(preopen: bool) -> None:
     wf = GenericNeXusWorkflow()
     wf[Filename[SampleRun]] = data.loki_tutorial_sample_run_60250()
     wf[NeXusName[Monitor1]] = 'monitor_1'
     wf[NeXusName[snx.NXdetector]] = 'larmor_detector'
+    wf[PreopenNeXusFile] = preopen
     da = wf.compute(DetectorData[SampleRun])
     assert 'position' in da.coords
     assert 'sample_position' in da.coords