essreduce 26.1.0__py3-none-any.whl → 26.2.0__py3-none-any.whl

ess/reduce/live/__init__.py

@@ -1,9 +1,3 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright (c) 2024 Scipp contributors (https://github.com/scipp)
 """Live data utilities"""
-
-from .workflow import LiveWorkflow
-
-__all__ = [
-    'LiveWorkflow',
-]

ess/reduce/streaming.py

@@ -5,12 +5,15 @@
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from copy import deepcopy
-from typing import Any, Generic, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar
 
 import networkx as nx
 import sciline
 import scipp as sc
 
+if TYPE_CHECKING:
+    import graphviz
+
 T = TypeVar('T')
 
 
@@ -108,6 +111,14 @@ class Accumulator(ABC, Generic[T]):
         Clear the accumulator, resetting it to its initial state.
         """
 
+    def on_finalize(self) -> None:
+        """
+        Called after finalize retrieves value.
+
+        Override this method to perform custom cleanup after each finalize cycle.
+        The default implementation does nothing.
+        """
+
 
 class EternalAccumulator(Accumulator[T]):
     """
@@ -352,11 +363,15 @@ class StreamProcessor:
         for key in context_keys:
             workflow[key] = None
 
+        # Store for visualization (copy in case caller modifies base_workflow later)
+        self._base_workflow_for_viz = base_workflow.copy()
+
         # Find and pre-compute static nodes as far down the graph as possible
         nodes = _find_descendants(workflow, dynamic_keys + context_keys)
         last_static = _find_parents(workflow, nodes) - nodes
         for key, value in base_workflow.compute(last_static).items():
             workflow[key] = value
+        self._cached_keys = last_static  # Store for visualization
 
         # Nodes that may need updating on context change but should be cached otherwise.
         dynamic_nodes = _find_descendants(workflow, dynamic_keys)
@@ -422,6 +437,9 @@ class StreamProcessor:
             needs_recompute |= self._context_key_to_cached_context_nodes_map[key]
         for key, value in context.items():
             self._context_workflow[key] = value
+            # Propagate context values to finalize workflow so providers that depend
+            # on context keys receive the updated values during finalize().
+            self._finalize_workflow[key] = value
         results = self._context_workflow.compute(needs_recompute)
         for key, value in results.items():
             if key in self._target_keys:
@@ -505,7 +523,10 @@ class StreamProcessor:
         """
         for key in self._accumulators:
             self._finalize_workflow[key] = self._accumulators[key].value
-        return self._finalize_workflow.compute(self._target_keys)
+        result = self._finalize_workflow.compute(self._target_keys)
+        for acc in self._accumulators.values():
+            acc.on_finalize()
+        return result
 
     def clear(self) -> None:
         """
@@ -517,15 +538,173 @@ class StreamProcessor:
         for accumulator in self._accumulators.values():
             accumulator.clear()
 
+    def _get_viz_workflow(self) -> sciline.Pipeline:
+        """Create the workflow used for visualization."""
+        viz_workflow = sciline.Pipeline()
+        for key in self._target_keys:
+            viz_workflow[key] = self._base_workflow_for_viz[key]
+        viz_graph = viz_workflow.underlying_graph
+        for key in self._dynamic_keys:
+            if key in viz_graph:
+                viz_workflow[key] = None
+        for key in self._context_keys:
+            if key in viz_graph:
+                viz_workflow[key] = None
+        return viz_workflow
+
+    def _classify_nodes(self, graph: nx.DiGraph) -> dict[str, set[Any]]:
+        """
+        Classify all nodes in the graph for visualization.
+
+        Node categories:
+
+        - static: Pre-computed once, not dependent on dynamic or context keys
+        - cached_keys: Subset of static nodes that are actually cached (last_static)
+        - dynamic_keys: Input entry points for chunk data
+        - dynamic_nodes: Downstream of dynamic keys, recomputed per chunk
+          (excludes nodes downstream of accumulators, which are computed in finalize)
+        - context_keys: Input entry points for context data
+        - context_dependent: Downstream of context keys but not dynamic keys
+        - accumulator_keys: Where values are aggregated across chunks
+        - target_keys: Final outputs computed in finalize()
+        - finalize_nodes: Downstream of accumulators, computed in finalize()
+        """
+        all_nodes = set(graph.nodes)
+        accumulator_keys = set(self._accumulators.keys())
+        target_keys = set(self._target_keys)
+
+        # Compute derived classifications
+        dynamic_descendants = _find_descendants(graph, self._dynamic_keys)
+        context_descendants = _find_descendants(graph, self._context_keys)
+
+        # Nodes downstream of accumulators are computed in finalize(), not per-chunk
+        accumulator_descendants = _find_descendants(graph, accumulator_keys)
+        finalize_nodes = accumulator_descendants - accumulator_keys
+
+        # Dynamic nodes: downstream of dynamic keys but NOT downstream of accumulators
+        # These are recomputed for each chunk
+        dynamic_nodes = (
+            dynamic_descendants - self._dynamic_keys - accumulator_descendants
+        )
+
+        # Context-dependent nodes: downstream of context but not of dynamic
+        context_dependent = (
+            context_descendants - dynamic_descendants - self._context_keys
+        )
+
+        # Static nodes: not dependent on dynamic or context
+        static_nodes = all_nodes - dynamic_descendants - context_descendants
+
+        return {
+            'static': static_nodes,
+            'cached_keys': self._cached_keys & all_nodes,
+            'dynamic_keys': self._dynamic_keys & all_nodes,
+            'dynamic_nodes': dynamic_nodes,
+            'context_keys': self._context_keys & all_nodes,
+            'context_dependent': context_dependent,
+            'accumulator_keys': accumulator_keys & all_nodes,
+            'target_keys': target_keys & all_nodes,
+            'finalize_nodes': finalize_nodes,
+        }
+
+    def visualize(
+        self,
+        compact: bool = False,
+        mode: Literal['data', 'task', 'both'] = 'data',
+        cluster_generics: bool = True,
+        cluster_color: str | None = '#f0f0ff',
+        show_legend: bool = True,
+        show_static_dependencies: bool = True,
+        **kwargs: Any,
+    ) -> 'graphviz.Digraph':
+        """
+        Visualize the workflow with node classification styling.
+
+        This post-processes sciline's visualization to add styling that highlights:
+
+        - Static nodes (gray): Pre-computed once, dependencies of cached nodes
+        - Static cached nodes (gray, thick border): Pre-computed and cached
+        - Dynamic keys (green, thick border): Input entry points for chunks
+        - Dynamic nodes (light green): Recomputed for each chunk
+        - Context keys (blue, thick border): Input entry points for context
+        - Context-dependent nodes (light blue): Cached until context changes
+        - Accumulator keys (orange cylinder): Aggregation points
+        - Finalize nodes (plum): Computed from accumulators during finalize
+        - Target keys (double border): Final outputs
+
+        Parameters
+        ----------
+        compact:
+            If True, parameter-table-dependent branches are collapsed.
+        mode:
+            'data' shows only data nodes, 'task' shows task nodes, 'both' shows all.
+        cluster_generics:
+            If True, generic products are grouped into clusters.
+        cluster_color:
+            Background color of clusters. If None, clusters are dotted.
+        show_legend:
+            If True, add a legend explaining the node styles.
+        show_static_dependencies:
+            If True (default), show all static nodes including dependencies of cached
+            nodes. If False, hide the ancestors of cached nodes to simplify the graph.
+        **kwargs:
+            Additional arguments passed to graphviz.Digraph.
+
+        Returns
+        -------
+        :
+            A graphviz.Digraph with styled nodes.
+        """
+        viz_workflow = self._get_viz_workflow()
+        if not show_static_dependencies:
+            # Create a pruned workflow that hides ancestors of cached nodes
+            viz_workflow = viz_workflow.copy()
+            for key in self._cached_keys:
+                if key in viz_workflow.underlying_graph:
+                    viz_workflow[key] = None
+
+        graph = viz_workflow.underlying_graph
+
+        dot = viz_workflow.visualize(
+            compact=compact,
+            mode=mode,
+            cluster_generics=cluster_generics,
+            cluster_color=cluster_color,
+            **kwargs,
+        )
+
+        classifications = self._classify_nodes(graph)
+
+        # Build a mapping from formatted names to original keys
+        key_to_formatted: dict[Any, str] = {}
+        for key in graph.nodes:
+            key_to_formatted[key] = _format_key_for_graphviz(key)
+
+        # Apply styles by re-adding nodes with updated attributes
+        for key, formatted_name in key_to_formatted.items():
+            style = _get_node_style(key, classifications)
+            if style:
+                dot.node(formatted_name, **style)
+
+        # Add legend
+        if show_legend:
+            _add_legend(dot, show_static_dependencies=show_static_dependencies)
+
+        return dot
+
 
 def _find_descendants(
-    workflow: sciline.Pipeline, keys: tuple[sciline.typing.Key, ...]
+    source: sciline.Pipeline | nx.DiGraph,
+    keys: set[sciline.typing.Key] | tuple[sciline.typing.Key, ...],
 ) -> set[sciline.typing.Key]:
-    graph = workflow.underlying_graph
+    """Find all descendants of any key in keys, including the keys themselves."""
+    graph = source.underlying_graph if hasattr(source, 'underlying_graph') else source
+    keys_set = set(keys)
     descendants = set()
-    for key in keys:
-        descendants |= nx.descendants(graph, key)
-    return descendants | set(keys)
+    for key in keys_set:
+        if key in graph:
+            descendants |= nx.descendants(graph, key)
+    return descendants | (keys_set & set(graph.nodes))
 
 
 def _find_parents(
@@ -536,3 +715,194 @@ def _find_parents(
     for key in keys:
         parents |= set(graph.predecessors(key))
     return parents
+
+
+# =============================================================================
+# Visualization helpers
+# =============================================================================
+
+# Style definitions for each node category
+# Priority order for overlapping categories (higher = takes precedence for fill)
+_VIZ_STYLES = {
+    'static': {
+        'fillcolor': '#e8e8e8',  # Gray
+        'style': 'filled',
+        'priority': 0,
+    },
+    'cached_keys': {
+        'fillcolor': '#e8e8e8',  # Gray (same as static)
+        'style': 'filled',
+        'penwidth': '2.5',  # Thick border to distinguish from dependencies
+        'priority': 1,
+    },
+    'context_dependent': {
+        'fillcolor': '#d4e8f4',  # Light blue
+        'style': 'filled',
+        'priority': 2,
+    },
+    'context_keys': {
+        'fillcolor': '#87CEEB',  # Sky blue
+        'style': 'filled',
+        'penwidth': '2.5',
+        'color': 'black',  # Override sciline's red for unsatisfied
+        'fontcolor': 'black',
+        'priority': 3,
+    },
+    'dynamic_nodes': {
+        'fillcolor': '#d4f4d4',  # Light green
+        'style': 'filled',
+        'priority': 4,
+    },
+    'dynamic_keys': {
+        'fillcolor': '#90EE90',  # Light green (stronger)
+        'style': 'filled',
+        'penwidth': '2.5',
+        'color': 'black',  # Override sciline's red for unsatisfied
+        'fontcolor': 'black',
+        'priority': 5,
+    },
+    'accumulator_keys': {
+        'fillcolor': '#FFB347',  # Orange
+        'style': 'filled',
+        'shape': 'cylinder',
+        'priority': 6,
+    },
+    'finalize_nodes': {
+        'fillcolor': '#DDA0DD',  # Plum (more distinct from cluster color)
+        'style': 'filled',
+        'priority': 7,
+    },
+    'target_keys': {
+        'peripheries': '2',  # Double border
+        'priority': 8,
+    },
+}
+
+
+def _format_key_for_graphviz(key: Any) -> str:
+    """Format a key to match sciline's node naming convention."""
+    from sciline.visualize import _format_type
+
+    return _format_type(key).name
+
+
+def _get_node_style(key: Any, classifications: dict[str, set[Any]]) -> dict[str, str]:
+    """
+    Determine the style for a node based on its classifications.
+
+    A node can belong to multiple categories. We combine styles with
+    higher priority categories taking precedence for conflicting attributes.
+    """
+    applicable = []
+    for category, keys in classifications.items():
+        if key in keys:
+            applicable.append((_VIZ_STYLES[category]['priority'], category))
+
+    if not applicable:
+        return {}
+
+    # Sort by priority and merge styles
+    applicable.sort()
+    merged: dict[str, str] = {}
+    for _, category in applicable:
+        style = _VIZ_STYLES[category].copy()
+        style.pop('priority')
+        merged.update(style)
+
+    return merged
+
+
+def _add_legend(
+    dot: 'graphviz.Digraph', *, show_static_dependencies: bool = True
+) -> None:
+    """Add a legend subgraph explaining the node styles."""
+    with dot.subgraph(name='cluster_legend') as legend:
+        legend.attr(label='Legend', fontsize='14', style='rounded')
+        legend.attr('node', shape='rectangle', width='1.5', height='0.3')
+
+        # Track first node for edge chaining
+        prev_node = None
+
+        if show_static_dependencies:
+            legend.node(
+                'legend_static',
+                'Static',
+                fillcolor='#e8e8e8',
+                style='filled',
+            )
+            prev_node = 'legend_static'
+
+        legend.node(
+            'legend_cached',
+            'Static (cached)',
+            fillcolor='#e8e8e8',
+            style='filled',
+            penwidth='2.5',
+        )
+        if prev_node:
+            legend.edge(prev_node, 'legend_cached', style='invis')
+        prev_node = 'legend_cached'
+
+        legend.node(
+            'legend_context_key',
+            'Context key (input)',
+            fillcolor='#87CEEB',
+            style='filled',
+            penwidth='2.5',
+        )
+        legend.edge(prev_node, 'legend_context_key', style='invis')
+        prev_node = 'legend_context_key'
+
+        legend.node(
+            'legend_context_dep',
+            'Context-dependent',
+            fillcolor='#d4e8f4',
+            style='filled',
+        )
+        legend.edge(prev_node, 'legend_context_dep', style='invis')
+        prev_node = 'legend_context_dep'
+
+        legend.node(
+            'legend_dynamic_key',
+            'Dynamic key (input)',
+            fillcolor='#90EE90',
+            style='filled',
+            penwidth='2.5',
+        )
+        legend.edge(prev_node, 'legend_dynamic_key', style='invis')
+        prev_node = 'legend_dynamic_key'
+
+        legend.node(
+            'legend_dynamic_node',
+            'Dynamic (per chunk)',
+            fillcolor='#d4f4d4',
+            style='filled',
+        )
+        legend.edge(prev_node, 'legend_dynamic_node', style='invis')
+        prev_node = 'legend_dynamic_node'
+
+        legend.node(
+            'legend_accumulator',
+            'Accumulator',
+            fillcolor='#FFB347',
+            style='filled',
+            shape='cylinder',
+        )
+        legend.edge(prev_node, 'legend_accumulator', style='invis')
+        prev_node = 'legend_accumulator'
+
+        legend.node(
+            'legend_finalize',
+            'Finalize (from accum.)',
+            fillcolor='#DDA0DD',
+            style='filled',
+        )
+        legend.edge(prev_node, 'legend_finalize', style='invis')
+        prev_node = 'legend_finalize'
+
+        legend.node(
+            'legend_target',
+            'Target (output)',
+            peripheries='2',
+        )
+        legend.edge(prev_node, 'legend_target', style='invis')

ess/reduce/time_of_flight/__init__.py

@@ -33,6 +33,8 @@ from .types import (
     TofLookupTable,
     TofLookupTableFilename,
     TofMonitor,
+    WavelengthDetector,
+    WavelengthMonitor,
 )
 from .workflow import GenericTofWorkflow
 
@@ -60,6 +62,8 @@ __all__ = [
     "TofLookupTableFilename",
     "TofLookupTableWorkflow",
     "TofMonitor",
+    "WavelengthDetector",
+    "WavelengthMonitor",
     "providers",
     "simulate_chopper_cascade_using_tof",
 ]

ess/reduce/time_of_flight/eto_to_tof.py

@@ -14,6 +14,7 @@ import scipp as sc
 import scippneutron as scn
 import scippnexus as snx
 from scippneutron._utils import elem_unit
+from scippneutron.conversion.tof import wavelength_from_tof
 
 try:
     from .interpolator_numba import Interpolator as InterpolatorImpl
@@ -39,6 +40,8 @@ from .types import (
     TofDetector,
     TofLookupTable,
     TofMonitor,
+    WavelengthDetector,
+    WavelengthMonitor,
 )
 
 
@@ -401,14 +404,15 @@ def _compute_tof_data(
 ) -> sc.DataArray:
     if da.bins is None:
         data = _time_of_flight_data_histogram(da=da, lookup=lookup, ltotal=ltotal)
-        return rebin_strictly_increasing(data, dim='tof')
+        out = rebin_strictly_increasing(data, dim='tof')
     else:
-        return _time_of_flight_data_events(
+        out = _time_of_flight_data_events(
             da=da,
             lookup=lookup,
             ltotal=ltotal,
             pulse_stride_offset=pulse_stride_offset,
         )
+    return out.assign_coords(Ltotal=ltotal)
 
 
 def detector_time_of_flight_data(
@@ -418,8 +422,9 @@ def detector_time_of_flight_data(
     pulse_stride_offset: PulseStrideOffset,
 ) -> TofDetector[RunType]:
     """
-    Convert the time-of-arrival data to time-of-flight data using a lookup table.
-    The output data will have a time-of-flight coordinate.
+    Convert the time-of-arrival (event_time_offset) data to time-of-flight data using a
+    lookup table.
+    The output data will have two new coordinates: time-of-flight and Ltotal.
 
     Parameters
     ----------
@@ -452,8 +457,9 @@ def monitor_time_of_flight_data(
     pulse_stride_offset: PulseStrideOffset,
 ) -> TofMonitor[RunType, MonitorType]:
     """
-    Convert the time-of-arrival data to time-of-flight data using a lookup table.
-    The output data will have a time-of-flight coordinate.
+    Convert the time-of-arrival (event_time_offset) data to time-of-flight data using a
+    lookup table.
+    The output data will have two new coordinates: time-of-flight and Ltotal.
 
     Parameters
     ----------
@@ -526,6 +532,47 @@ def detector_time_of_arrival_data(
     return ToaDetector[RunType](result)
 
 
+def _tof_to_wavelength(da: sc.DataArray) -> sc.DataArray:
+    """
+    Convert time-of-flight data to wavelength data.
+
+    Here we assume that the input data contains a Ltotal coordinate, which is required
+    for the conversion.
+    This coordinate is assigned in the ``_compute_tof_data`` function.
+    """
+    return da.transform_coords(
+        'wavelength', graph={"wavelength": wavelength_from_tof}, keep_intermediate=False
+    )
+
+
+def detector_wavelength_data(
+    detector_data: TofDetector[RunType],
+) -> WavelengthDetector[RunType]:
+    """
+    Convert time-of-flight coordinate of the detector data to wavelength.
+
+    Parameters
+    ----------
+    da:
+        Detector data with time-of-flight coordinate.
+    """
+    return WavelengthDetector[RunType](_tof_to_wavelength(detector_data))
+
+
+def monitor_wavelength_data(
+    monitor_data: TofMonitor[RunType, MonitorType],
+) -> WavelengthMonitor[RunType, MonitorType]:
+    """
+    Convert time-of-flight coordinate of the monitor data to wavelength.
+
+    Parameters
+    ----------
+    da:
+        Monitor data with time-of-flight coordinate.
+    """
+    return WavelengthMonitor[RunType, MonitorType](_tof_to_wavelength(monitor_data))
+
+
 def providers() -> tuple[Callable]:
     """
     Providers of the time-of-flight workflow.
@@ -536,4 +583,6 @@ def providers() -> tuple[Callable]:
         detector_ltotal_from_straight_line_approximation,
         monitor_ltotal_from_straight_line_approximation,
         detector_time_of_arrival_data,
+        detector_wavelength_data,
+        monitor_wavelength_data,
     )

ess/reduce/time_of_flight/lut.py

@@ -4,7 +4,6 @@
 Utilities for computing time-of-flight lookup tables from neutron simulations.
 """
 
-import math
 from dataclasses import dataclass
 from typing import NewType
 
@@ -17,11 +16,10 @@ from .types import TofLookupTable
 
 
 @dataclass
-class SimulationResults:
+class BeamlineComponentReading:
     """
-    Results of a time-of-flight simulation used to create a lookup table.
-
-    The results (apart from ``distance``) should be flat lists (1d arrays) of length N
+    Reading at a given position along the beamline from a time-of-flight simulation.
+    The data (apart from ``distance``) should be flat lists (1d arrays) of length N
     where N is the number of neutrons, containing the properties of the neutrons in the
     simulation.
 
@@ -30,9 +28,6 @@ class SimulationResults:
     time_of_arrival:
         Time of arrival of the neutrons at the position where the events were recorded
         (1d array of size N).
-    speed:
-        Speed of the neutrons, typically derived from the wavelength of the neutrons
-        (1d array of size N).
     wavelength:
         Wavelength of the neutrons (1d array of size N).
     weight:
@@ -43,15 +38,39 @@ class SimulationResults:
         For a ``tof`` simulation, this is just the position of the detector where the
         events are recorded. For a ``McStas`` simulation, this is the distance between
         the source and the event monitor.
-    choppers:
-        The parameters of the choppers used in the simulation (if any).
     """
 
     time_of_arrival: sc.Variable
-    speed: sc.Variable
     wavelength: sc.Variable
     weight: sc.Variable
     distance: sc.Variable
+
+    def __post_init__(self):
+        self.speed = (sc.constants.h / sc.constants.m_n) / self.wavelength
+
+
+@dataclass
+class SimulationResults:
+    """
+    Results of a time-of-flight simulation used to create a lookup table.
+    It should contain readings at various positions along the beamline, e.g., at
+    the source and after each chopper.
+    It also contains the chopper parameters used in the simulation, so it can be
+    determined if this simulation is compatible with a given experiment.
+
+    Parameters
+    ----------
+    readings:
+        A dict of :class:`BeamlineComponentReading` objects representing the readings at
+        various positions along the beamline. The keys in the dict should correspond to
+        the names of the components (e.g., 'source', 'chopper1', etc.).
+    choppers:
+        The chopper parameters used in the simulation (if any). These are used to verify
+        that the simulation is compatible with a given experiment (comparing chopper
+        openings, frequencies, phases, etc.).
+    """
+
+    readings: dict[str, BeamlineComponentReading]
     choppers: DiskChoppers[AnyRun] | None = None
 
 
@@ -145,11 +164,10 @@ def _mask_large_uncertainty(table: sc.DataArray, error_threshold: float):
     table.values[mask.values] = np.nan
 
 
-def _compute_mean_tof_in_distance_range(
-    simulation: SimulationResults,
-    distance_bins: sc.Variable,
+def _compute_mean_tof(
+    simulation: BeamlineComponentReading,
+    distance: sc.Variable,
     time_bins: sc.Variable,
-    distance_unit: str,
     time_unit: str,
     frame_period: sc.Variable,
     time_bins_half_width: sc.Variable,
@@ -162,12 +180,10 @@ def _compute_mean_tof_in_distance_range(
     ----------
     simulation:
         Results of a time-of-flight simulation used to create a lookup table.
-    distance_bins:
-        Bin edges for the distance axis in the lookup table.
+    distance:
+        Distance where table is computed.
     time_bins:
         Bin edges for the event_time_offset axis in the lookup table.
-    distance_unit:
-        Unit of the distance axis.
     time_unit:
         Unit of the event_time_offset axis.
     frame_period:
@@ -175,23 +191,17 @@ def _compute_mean_tof_in_distance_range(
     time_bins_half_width:
         Half width of the time bins in the event_time_offset axis.
     """
-    simulation_distance = simulation.distance.to(unit=distance_unit)
-    distances = sc.midpoints(distance_bins)
+    travel_length = distance - simulation.distance.to(unit=distance.unit)
     # Compute arrival and flight times for all neutrons
-    toas = simulation.time_of_arrival + (distances / simulation.speed).to(
+    toas = simulation.time_of_arrival + (travel_length / simulation.speed).to(
         unit=time_unit, copy=False
     )
-    dist = distances + simulation_distance
-    tofs = dist * (sc.constants.m_n / sc.constants.h) * simulation.wavelength
+    tofs = distance / simulation.speed
 
     data = sc.DataArray(
-        data=sc.broadcast(simulation.weight, sizes=toas.sizes),
-        coords={
-            "toa": toas,
-            "tof": tofs.to(unit=time_unit, copy=False),
-            "distance": dist,
-        },
-    ).flatten(to="event")
+        data=simulation.weight,
+        coords={"toa": toas, "tof": tofs.to(unit=time_unit, copy=False)},
+    )
 
     # Add the event_time_offset coordinate, wrapped to the frame_period
     data.coords['event_time_offset'] = data.coords['toa'] % frame_period
@@ -205,18 +215,14 @@ def _compute_mean_tof_in_distance_range(
     # data.coords['event_time_offset'] %= pulse_period - time_bins_half_width
     data.coords['event_time_offset'] %= frame_period - time_bins_half_width
 
-    binned = data.bin(
-        distance=distance_bins + simulation_distance, event_time_offset=time_bins
-    )
-
+    binned = data.bin(event_time_offset=time_bins)
+    binned_sum = binned.bins.sum()
     # Weighted mean of tof inside each bin
-    mean_tof = (
-        binned.bins.data * binned.bins.coords["tof"]
-    ).bins.sum() / binned.bins.sum()
+    mean_tof = (binned.bins.data * binned.bins.coords["tof"]).bins.sum() / binned_sum
     # Compute the variance of the tofs to track regions with large uncertainty
     variance = (
         binned.bins.data * (binned.bins.coords["tof"] - mean_tof) ** 2
-    ).bins.sum() / binned.bins.sum()
+    ).bins.sum() / binned_sum
 
     mean_tof.variances = variance.values
     return mean_tof
@@ -239,7 +245,7 @@ def make_tof_lookup_table(
     ----------
     simulation:
         Results of a time-of-flight simulation used to create a lookup table.
-        The results should be a flat table with columns for time-of-arrival, speed,
+        The results should be a flat table with columns for time-of-arrival,
         wavelength, and weight.
     ltotal_range:
         Range of total flight path lengths from the source to the detector.
@@ -294,15 +300,14 @@ def make_tof_lookup_table(
     left edge.
     """
     distance_unit = "m"
-    time_unit = simulation.time_of_arrival.unit
+    time_unit = "us"
     res = distance_resolution.to(unit=distance_unit)
     pulse_period = pulse_period.to(unit=time_unit)
     frame_period = pulse_period * pulse_stride
 
-    min_dist, max_dist = (
-        x.to(unit=distance_unit) - simulation.distance.to(unit=distance_unit)
-        for x in ltotal_range
-    )
+    min_dist = ltotal_range[0].to(unit=distance_unit)
+    max_dist = ltotal_range[1].to(unit=distance_unit)
+
     # We need to bin the data below, to compute the weighted mean of the wavelength.
     # This results in data with bin edges.
     # However, the 2d interpolator expects bin centers.
@@ -314,7 +319,7 @@ def make_tof_lookup_table(
     # ensure that the last bin is not cut off. We want the upper edge to be higher than
     # the maximum distance, hence we pad with an additional 1.5 x resolution.
     pad = 2.0 * res
-    distance_bins = sc.arange('distance', min_dist - pad, max_dist + pad, res)
+    distances = sc.arange('distance', min_dist - pad, max_dist + pad, res)
 
     # Create some time bins for event_time_offset.
     # We want our final table to strictly cover the range [0, frame_period].
@@ -329,22 +334,36 @@ def make_tof_lookup_table(
     time_bins_half_width = 0.5 * (time_bins[1] - time_bins[0])
     time_bins -= time_bins_half_width
 
-    # To avoid a too large RAM usage, we compute the table in chunks, and piece them
-    # together at the end.
-    ndist = len(distance_bins) - 1
-    max_size = 2e7
-    total_size = ndist * len(simulation.time_of_arrival)
-    nchunks = math.ceil(total_size / max_size)
-    chunk_size = math.ceil(ndist / nchunks)
+    # Sort simulation readings by reverse distance
+    sorted_simulation_results = sorted(
+        simulation.readings.values(), key=lambda x: x.distance.value, reverse=True
+    )
+
     pieces = []
-    for i in range(nchunks):
-        dist_edges = distance_bins[i * chunk_size : (i + 1) * chunk_size + 1]
+    # To avoid large RAM usage, and having to split the distances into chunks
+    # according to which component reading to use, we simply loop over distances one
+    # by one here.
+    for dist in distances:
+        # Find the correct simulation reading
+        simulation_reading = None
+        for reading in sorted_simulation_results:
+            if dist.value >= reading.distance.to(unit=dist.unit).value:
+                simulation_reading = reading
+                break
+        if simulation_reading is None:
+            closest = sorted_simulation_results[-1]
+            raise ValueError(
+                "Building the Tof lookup table failed: the requested position "
+                f"{dist.value} {dist.unit} is before the component with the lowest "
+                "distance in the simulation. The first component in the beamline "
+                f"has distance {closest.distance.value} {closest.distance.unit}."
+            )
+
         pieces.append(
-            _compute_mean_tof_in_distance_range(
-                simulation=simulation,
-                distance_bins=dist_edges,
+            _compute_mean_tof(
+                simulation=simulation_reading,
+                distance=dist,
                 time_bins=time_bins,
-                distance_unit=distance_unit,
                 time_unit=time_unit,
                 frame_period=frame_period,
                 time_bins_half_width=time_bins_half_width,
@@ -352,7 +371,6 @@ def make_tof_lookup_table(
         )
 
     table = sc.concat(pieces, 'distance')
-    table.coords["distance"] = sc.midpoints(table.coords["distance"])
     table.coords["event_time_offset"] = sc.midpoints(table.coords["event_time_offset"])
 
     # Copy the left edge to the right to create periodic boundary conditions
@@ -361,7 +379,7 @@ def make_tof_lookup_table(
             [table.data, table.data['event_time_offset', 0]], dim='event_time_offset'
         ),
         coords={
-            "distance": table.coords["distance"],
+            "distance": distances,
             "event_time_offset": sc.concat(
                 [table.coords["event_time_offset"], frame_period],
                 dim='event_time_offset',
@@ -388,6 +406,24 @@ def make_tof_lookup_table(
     )
 
 
+def _to_component_reading(component):
+    events = component.data.squeeze().flatten(to='event')
+    sel = sc.full(value=True, sizes=events.sizes)
+    for key in {'blocked_by_others', 'blocked_by_me'} & set(events.masks.keys()):
+        sel &= ~events.masks[key]
+    events = events[sel]
+    # If the component is a source, use 'birth_time' as 'toa'
+    toa = (
+        events.coords["toa"] if "toa" in events.coords else events.coords["birth_time"]
+    )
+    return BeamlineComponentReading(
+        time_of_arrival=toa,
+        wavelength=events.coords["wavelength"],
+        weight=events.data,
+        distance=component.distance,
+    )
+
+
 def simulate_chopper_cascade_using_tof(
     choppers: DiskChoppers[AnyRun],
     source_position: SourcePosition,
@@ -432,31 +468,14 @@ def simulate_chopper_cascade_using_tof(
     source = tof.Source(
         facility=facility, neutrons=neutrons, pulses=pulse_stride, seed=seed
     )
+    sim_readings = {"source": _to_component_reading(source)}
     if not tof_choppers:
-        events = source.data.squeeze().flatten(to='event')
-        return SimulationResults(
-            time_of_arrival=events.coords["birth_time"],
-            speed=events.coords["speed"],
-            wavelength=events.coords["wavelength"],
-            weight=events.data,
-            distance=0.0 * sc.units.m,
-        )
+        return SimulationResults(readings=sim_readings, choppers=None)
     model = tof.Model(source=source, choppers=tof_choppers)
     results = model.run()
-    # Find name of the furthest chopper in tof_choppers
-    furthest_chopper = max(tof_choppers, key=lambda c: c.distance)
-    events = results[furthest_chopper.name].data.squeeze().flatten(to='event')
-    events = events[
-        ~(events.masks["blocked_by_others"] | events.masks["blocked_by_me"])
-    ]
-    return SimulationResults(
-        time_of_arrival=events.coords["toa"],
-        speed=events.coords["speed"],
-        wavelength=events.coords["wavelength"],
-        weight=events.data,
-        distance=furthest_chopper.distance,
-        choppers=choppers,
-    )
+    for name, ch in results.choppers.items():
+        sim_readings[name] = _to_component_reading(ch)
+    return SimulationResults(readings=sim_readings, choppers=choppers)
 
 
 def TofLookupTableWorkflow():

ess/reduce/time_of_flight/types.py

@@ -87,3 +87,11 @@ class ToaDetector(sl.Scope[RunType, sc.DataArray], sc.DataArray):
 
 class TofMonitor(sl.Scope[RunType, MonitorType, sc.DataArray], sc.DataArray):
     """Monitor data with time-of-flight coordinate."""
+
+
+class WavelengthDetector(sl.Scope[RunType, sc.DataArray], sc.DataArray):
+    """Detector data with wavelength coordinate."""
+
+
+class WavelengthMonitor(sl.Scope[RunType, MonitorType, sc.DataArray], sc.DataArray):
+    """Monitor data with wavelength coordinate."""

ess/reduce/time_of_flight/workflow.py

@@ -7,11 +7,7 @@ import scipp as sc
 
 from ..nexus import GenericNeXusWorkflow
 from . import eto_to_tof
-from .types import (
-    PulseStrideOffset,
-    TofLookupTable,
-    TofLookupTableFilename,
-)
+from .types import PulseStrideOffset, TofLookupTable, TofLookupTableFilename
 
 
 def load_tof_lookup_table(

{essreduce-26.1.0.dist-info → essreduce-26.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: essreduce
-Version: 26.1.0
+Version: 26.2.0
 Summary: Common data reduction tools for the ESS facility
 Author: Scipp contributors
 License-Expression: BSD-3-Clause
@@ -25,6 +25,7 @@ Requires-Dist: scipp>=25.04.0
 Requires-Dist: scippneutron>=25.11.1
 Requires-Dist: scippnexus>=25.06.0
 Provides-Extra: test
+Requires-Dist: graphviz>=0.20; extra == "test"
 Requires-Dist: ipywidgets>=8.1; extra == "test"
 Requires-Dist: matplotlib>=3.10.7; extra == "test"
 Requires-Dist: numba>=0.59; extra == "test"

{essreduce-26.1.0.dist-info → essreduce-26.2.0.dist-info}/RECORD

@@ -3,16 +3,15 @@ ess/reduce/logging.py,sha256=6n8Czq4LZ3OK9ENlKsWSI1M3KvKv6_HSoUiV4__IUlU,357
 ess/reduce/normalization.py,sha256=r8H6SZgT94a1HE9qZ6Bx3N6c3VG3FzlJPzoCVMNI5-0,13081
 ess/reduce/parameter.py,sha256=4sCfoKOI2HuO_Q7JLH_jAXnEOFANSn5P3NdaOBzhJxc,4635
 ess/reduce/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ess/reduce/streaming.py,sha256=zbqxQz5dASDq4ZVyx-TdbapBXMyBttImCYz_6WOj4pg,17978
+ess/reduce/streaming.py,sha256=zdVosXbrtys5Z9gJ1dhnqoLeg2t8abicHBluytmrHCs,31150
 ess/reduce/ui.py,sha256=zmorAbDwX1cU3ygDT--OP58o0qU7OBcmJz03jPeYSLA,10884
 ess/reduce/uncertainty.py,sha256=LR4O6ApB6Z-W9gC_XW0ajupl8yFG-du0eee1AX_R-gk,6990
 ess/reduce/workflow.py,sha256=738-lcdgsORYfQ4A0UTk2IgnbVxC3jBdpscpaOFIpdc,3114
 ess/reduce/data/__init__.py,sha256=uDtqkmKA_Zwtj6II25zntz9T812XhdCn3tktYev4uyY,486
 ess/reduce/data/_registry.py,sha256=ngJMzP-AuMN0EKtws5vYSEPsv_Bn3TZjjIvNUKWQDeA,13992
-ess/reduce/live/__init__.py,sha256=jPQVhihRVNtEDrE20PoKkclKV2aBF1lS7cCHootgFgI,204
+ess/reduce/live/__init__.py,sha256=AFcqRbIhyqAo-oKV7gNIOfHRxfoVDiULf9vB4cEckHk,133
 ess/reduce/live/raw.py,sha256=z3JzKl1tOH51z1PWT3MJERactSFRXrNI_MBmpAHX71g,31094
 ess/reduce/live/roi.py,sha256=t65SfGtCtb8r-f4hkfg2I02CEnOp6Hh5Tv9qOqPOeK0,10588
-ess/reduce/live/workflow.py,sha256=bsbwvTqPhRO6mC__3b7MgU7DWwAnOvGvG-t2n22EKq8,4285
 ess/reduce/nexus/__init__.py,sha256=xXc982vZqRba4jR4z5hA2iim17Z7niw4KlS1aLFbn1Q,1107
 ess/reduce/nexus/_nexus_loader.py,sha256=5J26y_t-kabj0ik0jf3OLSYda3lDLDQhvPd2_ro7Q_0,23927
 ess/reduce/nexus/json_generator.py,sha256=ME2Xn8L7Oi3uHJk9ZZdCRQTRX-OV_wh9-DJn07Alplk,2529
@@ -20,15 +19,15 @@ ess/reduce/nexus/json_nexus.py,sha256=QrVc0p424nZ5dHX9gebAJppTw6lGZq9404P_OFl1gi
 ess/reduce/nexus/types.py,sha256=g5oBBEYPH7urF1tDP0tqXtixhQN8JDpe8vmiKrPiUW0,9320
 ess/reduce/nexus/workflow.py,sha256=bVRnVZ6HTEdIFwZv61JuvFUeTt9efUwe1MR65gBhyw8,24995
 ess/reduce/scripts/grow_nexus.py,sha256=hET3h06M0xlJd62E3palNLFvJMyNax2kK4XyJcOhl-I,3387
-ess/reduce/time_of_flight/__init__.py,sha256=bNxhK0uePltQpCW2sdNTpPdzXL6dt1IT1ri_cJ5VTL8,1561
-ess/reduce/time_of_flight/eto_to_tof.py,sha256=imKuN7IARMqBjmi8kjAcsseTFg6OD8ORas9X1FolgFY,18777
+ess/reduce/time_of_flight/__init__.py,sha256=LENpuVY2nP-YKVqUEPlmnvWdVaWBEKPinPITk-2crrA,1659
+ess/reduce/time_of_flight/eto_to_tof.py,sha256=aSQYzYW5rHhObGU9tu5UuZ-M0xfjnI0NdfJTTQa_-ww,20276
 ess/reduce/time_of_flight/fakes.py,sha256=BqpO56PQyO9ua7QlZw6xXMAPBrqjKZEM_jc-VB83CyE,4289
 ess/reduce/time_of_flight/interpolator_numba.py,sha256=wh2YS3j2rOu30v1Ok3xNHcwS7t8eEtZyZvbfXOCtgrQ,3835
 ess/reduce/time_of_flight/interpolator_scipy.py,sha256=_InoAPuMm2qhJKZQBAHOGRFqtvvuQ8TStoN7j_YgS4M,1853
-ess/reduce/time_of_flight/lut.py,sha256=8AupwtfB2983DoyzFgnRjWP3J3s_oPghi3XlXaaxxow,18768
+ess/reduce/time_of_flight/lut.py,sha256=JKZCbDYXOME6c5nMX01VvsAPU0JQgWTy64s2jZOI-7s,19746
 ess/reduce/time_of_flight/resample.py,sha256=Opmi-JA4zNH725l9VB99U4O9UlM37f5ACTCGtwBcows,3718
-ess/reduce/time_of_flight/types.py,sha256=FsSueM6OjJdF80uJHj-TNuyVAci8ixFvMuRMt9oHKDQ,3310
-ess/reduce/time_of_flight/workflow.py,sha256=2jUxeSmP0KweQTctAzIFJLm7Odf_e7kZzAc8MAMKBEs,3084
+ess/reduce/time_of_flight/types.py,sha256=Eb0l8_KlL7swORefRqbkMSCq6oxNeujU28-gSNXX9JU,3575
+ess/reduce/time_of_flight/workflow.py,sha256=adcEAwUrxx2I6IP-f0RNeUqDlMbE2i3s7evZBmmicpI,3067
 ess/reduce/widgets/__init__.py,sha256=SoSHBv8Dc3QXV9HUvPhjSYWMwKTGYZLpsWwsShIO97Q,5325
 ess/reduce/widgets/_base.py,sha256=_wN3FOlXgx_u0c-A_3yyoIH-SdUvDENGgquh9S-h5GI,4852
 ess/reduce/widgets/_binedges_widget.py,sha256=ZCQsGjYHnJr9GFUn7NjoZc1CdsnAzm_fMzyF-fTKKVY,2785
@@ -41,9 +40,9 @@ ess/reduce/widgets/_spinner.py,sha256=2VY4Fhfa7HMXox2O7UbofcdKsYG-AJGrsgGJB85nDX
 ess/reduce/widgets/_string_widget.py,sha256=iPAdfANyXHf-nkfhgkyH6gQDklia0LebLTmwi3m-iYQ,1482
 ess/reduce/widgets/_switchable_widget.py,sha256=fjKz99SKLhIF1BLgGVBSKKn3Lu_jYBwDYGeAjbJY3Q8,2390
 ess/reduce/widgets/_vector_widget.py,sha256=aTaBqCFHZQhrIoX6-sSqFWCPePEW8HQt5kUio8jP1t8,1203
-essreduce-26.1.0.dist-info/licenses/LICENSE,sha256=nVEiume4Qj6jMYfSRjHTM2jtJ4FGu0g-5Sdh7osfEYw,1553
-essreduce-26.1.0.dist-info/METADATA,sha256=WTZ9G8OVIHDjHsgeW9zgv-XJBv6mzYc5HAlxBY-5uXQ,1987
-essreduce-26.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-essreduce-26.1.0.dist-info/entry_points.txt,sha256=PMZOIYzCifHMTe4pK3HbhxUwxjFaZizYlLD0td4Isb0,66
-essreduce-26.1.0.dist-info/top_level.txt,sha256=0JxTCgMKPLKtp14wb1-RKisQPQWX7i96innZNvHBr-s,4
-essreduce-26.1.0.dist-info/RECORD,,
+essreduce-26.2.0.dist-info/licenses/LICENSE,sha256=nVEiume4Qj6jMYfSRjHTM2jtJ4FGu0g-5Sdh7osfEYw,1553
+essreduce-26.2.0.dist-info/METADATA,sha256=BGHgAZ8pZ7iSElnZZ0eAFlUZQhxk1xjg8ivigzMu6h8,2034
+essreduce-26.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+essreduce-26.2.0.dist-info/entry_points.txt,sha256=PMZOIYzCifHMTe4pK3HbhxUwxjFaZizYlLD0td4Isb0,66
+essreduce-26.2.0.dist-info/top_level.txt,sha256=0JxTCgMKPLKtp14wb1-RKisQPQWX7i96innZNvHBr-s,4
+essreduce-26.2.0.dist-info/RECORD,,

{essreduce-26.1.0.dist-info → essreduce-26.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ess/reduce/live/workflow.py

@@ -1,128 +0,0 @@
-# SPDX-License-Identifier: BSD-3-Clause
-# Copyright (c) 2024 Scipp contributors (https://github.com/scipp)
-"""Tools for creating live data reduction workflows for Beamlime."""
-
-from pathlib import Path
-from typing import NewType, TypeVar
-
-import sciline
-import scipp as sc
-import scippnexus as snx
-
-from ess.reduce import streaming
-from ess.reduce.nexus import types as nt
-from ess.reduce.nexus.json_nexus import JSONGroup
-
-JSONEventData = NewType('JSONEventData', dict[str, JSONGroup])
-
-
-def _load_json_event_data(name: str, nxevent_data: JSONEventData) -> sc.DataArray:
-    return snx.Group(nxevent_data[name], definitions=snx.base_definitions())[()]
-
-
-def load_json_event_data_for_sample_run(
-    name: nt.NeXusName[nt.Component], nxevent_data: JSONEventData
-) -> nt.NeXusData[nt.Component, nt.SampleRun]:
-    return nt.NeXusData[nt.Component, nt.SampleRun](
-        _load_json_event_data(name, nxevent_data)
-    )
-
-
-def load_json_event_data_for_sample_transmission_run(
-    name: nt.NeXusName[nt.Component], nxevent_data: JSONEventData
-) -> nt.NeXusData[nt.Component, nt.TransmissionRun[nt.SampleRun]]:
-    return nt.NeXusData[nt.Component, nt.TransmissionRun[nt.SampleRun]](
-        _load_json_event_data(name, nxevent_data)
-    )
-
-
-T = TypeVar('T', bound='LiveWorkflow')
-
-
-class LiveWorkflow:
-    """A workflow class that fulfills Beamlime's LiveWorkflow protocol."""
-
-    def __init__(
-        self,
-        *,
-        streamed: streaming.StreamProcessor,
-        outputs: dict[str, sciline.typing.Key],
-    ) -> None:
-        self._streamed = streamed
-        self._outputs = outputs
-
-    @classmethod
-    def from_workflow(
-        cls: type[T],
-        *,
-        workflow: sciline.Pipeline,
-        accumulators: dict[sciline.typing.Key, streaming.Accumulator],
-        outputs: dict[str, sciline.typing.Key],
-        run_type: type[nt.RunType],
-        nexus_filename: Path,
-    ) -> T:
-        """
-        Create a live workflow from a base workflow and other parameters.
-
-        Parameters
-        ----------
-        workflow:
-            Base workflow to use for live data reduction.
-        accumulators:
-            Accumulators forwarded to the stream processor.
-        outputs:
-            Mapping from output names to keys in the workflow. The keys correspond to
-            workflow results that will be computed.
-        run_type:
-            Type of the run to process. This defines which run is the dynamic run being
-            processed. The NeXus template file will be set as the filename for this run.
-        nexus_filename:
-            Path to the NeXus file to process.
-
-        Returns
-        -------
-        :
-            Live workflow object.
-        """
-
-        workflow = workflow.copy()
-        if run_type is nt.SampleRun:
-            workflow.insert(load_json_event_data_for_sample_run)
-        elif run_type is nt.TransmissionRun[nt.SampleRun]:
-            workflow.insert(load_json_event_data_for_sample_transmission_run)
-        else:
-            raise NotImplementedError(f"Run type {run_type} not supported yet.")
-        workflow[nt.Filename[run_type]] = nexus_filename
-        streamed = streaming.StreamProcessor(
-            base_workflow=workflow,
-            dynamic_keys=(JSONEventData,),
-            target_keys=outputs.values(),
-            accumulators=accumulators,
-        )
-        return cls(streamed=streamed, outputs=outputs)
-
-    def __call__(
-        self, nxevent_data: dict[str, JSONGroup], nxlog: dict[str, JSONGroup]
-    ) -> dict[str, sc.DataArray]:
-        """
-        Implements the __call__ method required by the LiveWorkflow protocol.
-
-        Parameters
-        ----------
-        nxevent_data:
-            NeXus event data.
-        nxlog:
-            NeXus log data. WARNING: This is currently not used.
-
-        Returns
-        -------
-        :
-            Dictionary of computed and plottable results.
-        """
-        # Beamlime passes full path, but the workflow only needs the name of the monitor
-        # or detector group.
-        nxevent_data = {
-            key.lstrip('/').split('/')[2]: value for key, value in nxevent_data.items()
-        }
-        results = self._streamed.add_chunk({JSONEventData: nxevent_data})
-        return {name: results[key] for name, key in self._outputs.items()}