pytesprocess 0.1.1__py3-none-any.whl

pytesprocess/process/ivprocess.py

@@ -0,0 +1,1380 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+import hashlib
+from itertools import repeat
+from multiprocessing import Pool
+from pathlib import Path
+import numpy as np
+import pandas as pd
+import qetpy as qp
+
+from pytesdaqx.io import AcquisitionCatalog, StreamReader
+from pytesprocess.core import FilterData
+from pytesprocess.process.randoms import Randoms
+from pytesprocess.utils import find_linear_segment
+
+
+__all__ = ["IVSweepProcessing"]
+
+
+@dataclass(frozen=True, slots=True)
+class _SweepPoint:
+    """Storage-independent description of one IV or dIdV sweep stream."""
+
+    acquisition_path: str
+    acquisition_name: str
+    measurement_type: str
+    stream_id: str
+    stream_num: int | None
+    stream_name: str | None
+    storage_format: str
+    sample_rate_hz: float
+    bias_ua: float
+    scan_index: int | None
+    sequence_index: int | None
+    sequence_repeat_index: int | None
+    adc_mode: str | None
+    raw_shape_model: str | None
+    duration_s: float
+    n_samples: int | None
+    hdf5_dump_segments: tuple[tuple[int, int, int], ...] = ()
+    trace_length_samples: int | None = None
+    n_traces: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class _SweepPair:
+    iv: _SweepPoint
+    didv: _SweepPoint
+    bias_delta_ua: float
+    match_method: str
+
+
+class IVSweepProcessing:
+    """Process IV/dIdV sweeps from HDF5 and Zarr acquisitions.
+
+    Sweep discovery is catalog-driven. New acquisitions use explicit
+    ``scan`` metadata when available; legacy acquisitions fall back to
+    identifying detector channels whose TES DC bias changes across streams.
+
+    IV trace handling is storage independent:
+      * legacy HDF5 with native records equal to the requested IV trace length
+        is read directly;
+      * longer HDF5 segments are sampled with :class:`Randoms` and read with
+        triggered ``StreamReader.read_records`` calls;
+      * continuous Zarr streams are sampled across the complete stream (no
+        processing partition required) and read the same way.
+
+    dIdV processing continues to use native finite records.
+    """
+
+    def __init__(
+        self,
+        data_paths,
+        processing_label=None,
+        bias_tolerance_percent=0.1,
+        bias_tolerance_ua=0.001,
+        verbose=True,
+    ):
+        self._processing_label = processing_label
+        self._verbose = bool(verbose)
+        self._bias_tolerance_percent = float(bias_tolerance_percent)
+        self._bias_tolerance_ua = float(bias_tolerance_ua)
+
+        discovery = self._discover_sweep_data(data_paths)
+        self._raw_data_dict = discovery["data"]
+        self._acquisition_name_iv = discovery["acquisition_name_iv"]
+        self._acquisition_name_didv = discovery["acquisition_name_didv"]
+        self._base_path_iv = discovery["base_path_iv"]
+        self._base_path_didv = discovery["base_path_didv"]
+
+        self.describe()
+        self._filter_data = FilterData()
+
+    def describe(self):
+        """Describe available IV/dIdV sweep points."""
+
+        print("\nIV/dIdV sweep available data:")
+        for channel, channel_data in self._raw_data_dict.items():
+            print(f"\n{channel}:")
+
+            iv_points = channel_data.get("IV") or []
+            didv_points = channel_data.get("dIdV") or []
+            common_points = channel_data.get("IV_dIdV") or []
+
+            if iv_points:
+                print(f" -IV: {len(iv_points)} bias points")
+            if didv_points:
+                print(f" -dIdV: {len(didv_points)} bias points")
+            if common_points:
+                print(f" -Common IV-dIdV: {len(common_points)} bias points")
+            elif iv_points and didv_points:
+                print(" -Common IV-dIdV: No bias points")
+
+    def process(
+        self,
+        channels=None,
+        enable_iv=True,
+        enable_didv=True,
+        trace_length_iv_msec=100,
+        nrandoms_iv=None,
+        min_separation_iv_msec=None,
+        random_seed=None,
+        lgc_output=True,
+        lgc_save=False,
+        save_path=None,
+        ncores=1,
+    ):
+        """Process selected IV/dIdV sweep channels.
+
+        Parameters
+        ----------
+        channels : str or list[str], optional
+            Detector channels. ``None`` processes every discovered sweep
+            channel.
+        enable_iv, enable_didv : bool
+            Enable IV and/or dIdV processing.
+        trace_length_iv_msec : float
+            IV noise trace length. The default is 100 ms, matching the legacy
+            HDF5 IV segment length.
+        nrandoms_iv : int, optional
+            Number of randomly selected IV traces per continuous bias-point
+            stream. ``None`` uses the maximum count allowed by the edge and
+            minimum-separation constraints, which approximates using all
+            available data without overlap.
+        min_separation_iv_msec : float, optional
+            Minimum trigger separation for randomly selected continuous IV
+            traces. ``None`` defaults to ``trace_length_iv_msec`` so selected
+            traces do not overlap.
+        random_seed : int, optional
+            Base seed for reproducible IV random selection. A stable per-stream
+            seed is derived from this value so results do not depend on the
+            multiprocessing split.
+        lgc_save : bool
+            Save the resulting FilterData HDF5 file.
+        lgc_output : bool
+            Return ``dict[channel, pandas.DataFrame]``.
+        save_path : str, optional
+            Output directory base.
+        ncores : int
+            Number of processes used across bias points for each channel.
+        """
+
+        if not enable_iv and not enable_didv:
+            raise ValueError("ERROR: You need to enable IV or dIdV!")
+        if float(trace_length_iv_msec) <= 0:
+            raise ValueError("trace_length_iv_msec must be positive")
+        if int(ncores) <= 0:
+            raise ValueError("ncores must be a positive integer")
+        if nrandoms_iv is not None and int(nrandoms_iv) <= 0:
+            raise ValueError("nrandoms_iv must be positive when provided")
+        if min_separation_iv_msec is None:
+            min_separation_iv_msec = float(trace_length_iv_msec)
+        if float(min_separation_iv_msec) < float(trace_length_iv_msec):
+            if self._verbose:
+                print(
+                    "WARNING: min_separation_iv_msec is shorter than the IV "
+                    "trace length, so randomly selected IV traces may overlap."
+                )
+
+        if channels is None:
+            channels = list(self._raw_data_dict)
+            if not channels:
+                raise ValueError("ERROR: No channels available!")
+        elif isinstance(channels, str):
+            channels = [channels]
+        else:
+            channels = list(channels)
+
+        for channel in channels:
+            if channel not in self._raw_data_dict:
+                raise ValueError(f"ERROR: channel {channel} not available!")
+
+        output_dict = {}
+
+        for channel in channels:
+            if self._verbose:
+                print(f"INFO: Channel {channel} IV and/or dIdV processing")
+
+            channel_data = self._raw_data_dict[channel]
+            channel_enable_iv = bool(enable_iv and channel_data.get("IV"))
+            channel_enable_didv = bool(enable_didv and channel_data.get("dIdV"))
+
+            if not channel_enable_iv and not channel_enable_didv:
+                raise ValueError(
+                    f"ERROR: No requested IV or dIdV data found for channel {channel}."
+                )
+
+            if channel_enable_iv and channel_enable_didv:
+                work_items = list(channel_data.get("IV_dIdV") or [])
+                processing_type = "IV_dIdV"
+                if not work_items:
+                    raise ValueError(
+                        f"ERROR: Unable to process both IV and dIdV for channel "
+                        f"{channel}. No common bias points were found. Adjust "
+                        "bias_tolerance_percent/bias_tolerance_ua if appropriate."
+                    )
+            elif channel_enable_iv:
+                work_items = list(channel_data["IV"])
+                processing_type = "IV"
+            else:
+                work_items = list(channel_data["dIdV"])
+                processing_type = "dIdV"
+
+            if not work_items:
+                raise ValueError(f"ERROR: No sweep points found for channel {channel}.")
+
+            n_workers = max(1, min(int(ncores), len(work_items)))
+            chunks = self._split_work_items(work_items, n_workers)
+
+            worker_args = zip(
+                chunks,
+                repeat(channel),
+                repeat(processing_type),
+                repeat(float(trace_length_iv_msec)),
+                repeat(None if nrandoms_iv is None else int(nrandoms_iv)),
+                repeat(float(min_separation_iv_msec)),
+                repeat(random_seed),
+            )
+
+            if n_workers == 1:
+                output_channel_df = self._process_work_items(*next(worker_args))
+            else:
+                if self._verbose:
+                    print(
+                        f"INFO: Processing {len(work_items)} bias points with "
+                        f"{n_workers} processes"
+                    )
+                with Pool(processes=n_workers) as pool:
+                    df_list = pool.starmap(self._process_work_items, worker_args)
+                output_channel_df = pd.concat(df_list, ignore_index=True)
+
+            output_channel_df = output_channel_df.sort_values(
+                "tes_bias", ascending=False, key=np.abs, ignore_index=True
+            )
+            self._assign_sweep_state(output_channel_df)
+            output_dict[channel] = output_channel_df
+
+            if self._verbose:
+                counts = output_channel_df["state"].value_counts(dropna=False)
+                n_normal = int(counts.get("normal", 0))
+                n_sc = int(counts.get("sc", 0))
+                print(f"INFO: IV/dIdV processing done for channel {channel}!")
+                if n_sc:
+                    print(f"INFO: Found {n_sc} SC points based on linearity")
+                else:
+                    print("INFO: Unable to estimate SC points based on linearity")
+                if n_normal:
+                    print(f"INFO: Found {n_normal} normal points based on linearity")
+                else:
+                    print("INFO: Unable to estimate normal points based on linearity")
+
+        self._filter_data.set_ivsweep_data_from_dict(output_dict)
+
+        if lgc_save:
+            processed_iv = any("offset_iv" in df.columns for df in output_dict.values())
+            file_name = self._save_filter_data(
+                save_path, prefer_iv=processed_iv
+            )
+            print(f"INFO: Saving dataframe in {file_name}")
+
+        if self._verbose:
+            print("INFO: IV/dIdV processing done!")
+
+        if lgc_output:
+            return output_dict
+        return None
+
+    def plot_ivsweep_offset(self, channel, tag="default"):
+        self._filter_data.plot_ivsweep_offset(channel=channel, tag=tag)
+
+    def _process_work_items(
+        self,
+        work_items,
+        channel,
+        processing_type,
+        trace_length_iv_msec,
+        nrandoms_iv,
+        min_separation_iv_msec,
+        random_seed,
+    ):
+        rows = []
+
+        for item in work_items:
+            if processing_type == "IV_dIdV":
+                iv_point = item.iv
+                didv_point = item.didv
+                pair = item
+            elif processing_type == "IV":
+                iv_point = item
+                didv_point = None
+                pair = None
+            else:
+                iv_point = None
+                didv_point = item
+                pair = None
+
+            display_bias = iv_point.bias_ua if iv_point is not None else didv_point.bias_ua
+            if self._verbose:
+                print(f"INFO: processing channel {channel}, bias point {display_bias} uA")
+
+            row = {
+                "channel": channel,
+                "processing_type": processing_type,
+                "processing_label": self._processing_label,
+                "tes_bias_ua": float(display_bias),
+            }
+
+            first_result = None
+            if iv_point is not None:
+                iv_result = self._process_measurement_point(
+                    iv_point,
+                    channel=channel,
+                    measurement_type="iv",
+                    trace_length_iv_msec=trace_length_iv_msec,
+                    nrandoms_iv=nrandoms_iv,
+                    min_separation_iv_msec=min_separation_iv_msec,
+                    random_seed=random_seed,
+                )
+                row.update(iv_result)
+                first_result = iv_result
+
+            if didv_point is not None:
+                didv_result = self._process_measurement_point(
+                    didv_point,
+                    channel=channel,
+                    measurement_type="didv",
+                    trace_length_iv_msec=trace_length_iv_msec,
+                    nrandoms_iv=nrandoms_iv,
+                    min_separation_iv_msec=min_separation_iv_msec,
+                    random_seed=random_seed,
+                )
+                row.update(didv_result)
+                if first_result is None:
+                    first_result = didv_result
+
+            row["tes_bias"] = float(first_result["_tes_bias"])
+            row["rshunt"] = float(first_result["_rshunt"])
+            row["rp"] = first_result["_rp"]
+            row["temperature_mc"] = first_result["_temperature_mc"]
+            row["temperature_cp"] = first_result["_temperature_cp"]
+            row["temperature_still"] = first_result["_temperature_still"]
+
+            if iv_point is not None:
+                row["tes_bias_point_iv_ua"] = float(iv_point.bias_ua)
+            if didv_point is not None:
+                row["tes_bias_point_didv_ua"] = float(didv_point.bias_ua)
+            if pair is not None:
+                row["bias_match_delta_ua"] = float(pair.bias_delta_ua)
+                row["bias_match_method"] = pair.match_method
+
+            for key in list(row):
+                if key.startswith("_"):
+                    row.pop(key)
+            rows.append(row)
+
+        return pd.DataFrame(rows)
+
+    def _process_measurement_point(
+        self,
+        point,
+        *,
+        channel,
+        measurement_type,
+        trace_length_iv_msec,
+        nrandoms_iv,
+        min_separation_iv_msec,
+        random_seed,
+    ):
+        traces, infos, detector_settings, trace_selection = self._read_point_traces(
+            point,
+            channel=channel,
+            measurement_type=measurement_type,
+            trace_length_iv_msec=trace_length_iv_msec,
+            nrandoms_iv=nrandoms_iv,
+            min_separation_iv_msec=min_separation_iv_msec,
+            random_seed=random_seed,
+        )
+
+        traces = np.asarray(traces)
+        if traces.ndim != 3 or traces.shape[1] != 1:
+            raise ValueError(
+                f"Expected traces with shape (n_traces, 1, n_samples) for "
+                f"{channel}, got {traces.shape}."
+            )
+        traces = traces[:, 0, :]
+
+        if traces.shape[0] == 0:
+            raise ValueError(
+                f"No {measurement_type} traces available for channel {channel}, "
+                f"stream {point.stream_id}."
+            )
+
+        fs = float(infos[0].get("sample_rate_hz", point.sample_rate_hz)) if infos else float(point.sample_rate_hz)
+        settings = detector_settings[channel]
+
+        tes_bias = self._finite_float(settings.get("tes_bias_dc_amps"), "tes_bias_dc_amps", channel)
+        output_gain = self._float_or_nan(settings.get("output_gain"))
+        close_loop_norm = self._float_or_nan(settings.get("close_loop_norm"))
+        output_offset = self._float_or_nan(settings.get("output_offset_vdc"))
+        rshunt = self._finite_float(settings.get("shunt_resistance_ohm"), "shunt_resistance_ohm", channel)
+        rp = self._float_or_nan(
+            settings.get("parasitic_resistance_ohm", settings.get("rp"))
+        )
+        sgamp = self._float_or_nan(settings.get("tes_bias_ac_amplitude_amps"))
+        sgfreq = self._float_or_nan(settings.get("tes_bias_ac_frequency_hz"))
+        dutycycle = self._float_or_default(
+            settings.get(
+                "dutycycle",
+                settings.get("duty_cycle", settings.get("tes_bias_ac_dutycycle")),
+            ),
+            0.5,
+        )
+
+        acquisition_name = (
+            infos[0].get("acquisition_name") if infos else None
+        ) or point.acquisition_name
+        stream_name = (
+            infos[0].get("stream_name") if infos else None
+        ) or point.stream_name or point.stream_id
+
+        # Reject only traces that are entirely zero. The previous implementation
+        # accidentally rejected every trace containing even a single exact zero.
+        nonzero = ~np.all(traces == 0, axis=1)
+        traces = traces[nonzero]
+        if traces.shape[0] == 0:
+            raise ValueError(
+                f"All {measurement_type} traces are zero for channel {channel}, "
+                f"stream {point.stream_id}."
+            )
+
+        result = {
+            f"stream_id_{measurement_type}": str(point.stream_id),
+            f"stream_name_{measurement_type}": str(stream_name),
+            f"acquisition_name_{measurement_type}": str(acquisition_name),
+            f"fs_{measurement_type}": fs,
+            f"output_variable_gain_{measurement_type}": output_gain,
+            f"output_variable_offset_{measurement_type}": output_offset,
+            f"close_loop_norm_{measurement_type}": close_loop_norm,
+            f"rshunt_{measurement_type}": rshunt,
+            f"rp_{measurement_type}": rp,
+            f"tes_bias_{measurement_type}": tes_bias,
+            f"ntraces_{measurement_type}": int(traces.shape[0]),
+            f"trace_length_samples_{measurement_type}": int(traces.shape[-1]),
+            f"trace_selection_{measurement_type}": trace_selection,
+            "_tes_bias": tes_bias,
+            "_rshunt": rshunt,
+            "_rp": rp,
+            "_temperature_mc": self._temperature_value(settings, "mc"),
+            "_temperature_cp": self._temperature_value(settings, "cp"),
+            "_temperature_still": self._temperature_value(settings, "still"),
+        }
+
+        if measurement_type == "iv":
+            cut = np.asarray(qp.autocuts_noise(traces, fs=fs), dtype=bool)
+        else:
+            cut = np.asarray(qp.autocuts_didv(traces, fs=fs), dtype=bool)
+
+        if cut.size != traces.shape[0]:
+            raise ValueError(
+                f"Unexpected autocut size {cut.size} for {traces.shape[0]} traces."
+            )
+        n_pass = int(np.sum(cut))
+        if n_pass == 0:
+            raise ValueError(
+                f"No {measurement_type} traces survive autocuts for channel "
+                f"{channel}, stream {point.stream_id}."
+            )
+
+        cut_eff = n_pass / len(cut)
+        selected = traces[cut]
+
+        if measurement_type == "iv":
+            psd_freq, psd = qp.calc_psd(selected, fs=fs, folded_over=False)
+            offset, offset_err = qp.utils.calc_offset(selected, fs=fs)
+            avgtrace = np.mean(selected, axis=0)
+            result.update({"psd": psd, "psd_freq": psd_freq})
+        else:
+            if not np.isfinite(sgfreq) or sgfreq <= 0:
+                raise ValueError(
+                    f"Invalid dIdV frequency {sgfreq!r} for channel {channel}, "
+                    f"stream {point.stream_id}."
+                )
+            if not np.isfinite(sgamp):
+                raise ValueError(
+                    f"Invalid dIdV amplitude {sgamp!r} for channel {channel}, "
+                    f"stream {point.stream_id}."
+                )
+
+            offset, offset_err = qp.utils.calc_offset(
+                selected, fs=fs, sgfreq=sgfreq, is_didv=True
+            )
+            avgtrace = np.mean(selected, axis=0)
+
+            avgtrace_lp = qp.utils.lowpassfilter(
+                avgtrace, cut_off_freq=1.5e3, fs=fs, order=1
+            )
+            nb_bins = len(avgtrace_lp)
+            nb_cycles = (nb_bins / fs) * sgfreq
+            start_bin = round(nb_bins * 0.1)
+            end_bin = round(nb_bins * 0.9)
+            if nb_cycles > 1.5:
+                start_bin = round(fs / sgfreq * 0.25)
+                end_bin = min(nb_bins, 4 * start_bin)
+
+            calc_values = avgtrace_lp[start_bin:end_bin]
+            if calc_values.size == 0:
+                rtes_estimate = np.nan
+            else:
+                delta_i = float(calc_values.max() - calc_values.min())
+                delta_v = sgamp * rshunt
+                rtes_estimate = np.nan if delta_i == 0 else delta_v / delta_i * 1e3
+
+            didvobj = qp.DIDV(
+                selected,
+                fs,
+                sgfreq,
+                sgamp,
+                rshunt,
+                autoresample=False,
+                dutycycle=dutycycle,
+            )
+            didvobj.processtraces()
+
+            result.update(
+                {
+                    "sgamp": sgamp,
+                    "sgfreq": sgfreq,
+                    "didvmean": didvobj._didvmean,
+                    "didvstd": didvobj._didvstd,
+                    "dutycycle": dutycycle,
+                    "rtes_estimate": rtes_estimate,
+                }
+            )
+
+        result.update(
+            {
+                f"offset_{measurement_type}": offset,
+                f"offset_err_{measurement_type}": offset_err,
+                f"cut_eff_{measurement_type}": cut_eff,
+                f"cut_{measurement_type}": cut,
+                f"cut_pass_{measurement_type}": True,
+                f"avgtrace_{measurement_type}": avgtrace,
+            }
+        )
+        return result
+
+    def _read_point_traces(
+        self,
+        point,
+        *,
+        channel,
+        measurement_type,
+        trace_length_iv_msec,
+        nrandoms_iv,
+        min_separation_iv_msec,
+        random_seed,
+    ):
+        """Read native or randomly selected records for one sweep point."""
+
+        with StreamReader(
+            point.acquisition_path,
+            streams=point.stream_id,
+            measurement_types=measurement_type,
+            verbose=False,
+        ) as reader:
+            detector_settings = reader.get_detector_settings()
+            if channel not in detector_settings:
+                raise ValueError(
+                    f"Channel {channel} is not available in stream {point.stream_id}."
+                )
+
+            if measurement_type == "didv":
+                # HDF5 dIdV is historically stored as finite segments even when
+                # adc_mode metadata says "continuous". Only native Zarr
+                # channel-sample streams are truly continuous here.
+                if point.storage_format == "zarr" and reader.is_continuous_stream:
+                    raise ValueError(
+                        f"Continuous dIdV stream {point.stream_id} is not supported "
+                        "by IVSweepProcessing yet; dIdV requires finite native records."
+                    )
+                traces, infos = reader.read_records(
+                    channels=channel,
+                    units="amps",
+                    include_metadata=True,
+                    stack=True,
+                )
+                return traces, infos, detector_settings, "native_records"
+
+            fs = float(reader.sample_rate_hz)
+            trace_length_samples = int(
+                round(float(trace_length_iv_msec) * fs / 1000.0)
+            )
+            if trace_length_samples <= 0:
+                raise ValueError("Requested IV trace length converts to zero samples")
+            pretrigger_samples = trace_length_samples // 2
+            edge_exclusion_msec = pretrigger_samples / fs * 1000.0
+            min_separation_samples = int(
+                np.ceil(float(min_separation_iv_msec) * fs / 1000.0)
+            )
+
+            if point.storage_format == "hdf5":
+                native_lengths = {
+                    int(length)
+                    for _, _, length in point.hdf5_dump_segments
+                    if int(length) > 0
+                }
+                if not native_lengths:
+                    raise ValueError(
+                        f"Unable to determine HDF5 segment length for stream {point.stream_id}."
+                    )
+                if len(native_lengths) != 1:
+                    raise ValueError(
+                        f"HDF5 stream {point.stream_id} contains mixed segment lengths "
+                        f"{sorted(native_lengths)}; IVSweepProcessing expects one native "
+                        "segment length per sweep point."
+                    )
+                native_length = next(iter(native_lengths))
+
+                if native_length == trace_length_samples:
+                    record_list = None
+                    n_records = None
+                    selection = "native_hdf5_segments"
+                    if nrandoms_iv is not None:
+                        record_list = self._random_native_hdf5_records(
+                            point,
+                            int(nrandoms_iv),
+                            seed=self._point_seed(
+                                random_seed, channel, measurement_type, point
+                            ),
+                        )
+                        selection = "random_native_hdf5_segments"
+                    traces, infos = reader.read_records(
+                        record_list=record_list,
+                        n_records=n_records,
+                        channels=channel,
+                        units="amps",
+                        include_metadata=True,
+                        stack=True,
+                    )
+                    return traces, infos, detector_settings, selection
+
+                if native_length < trace_length_samples:
+                    raise ValueError(
+                        f"Requested IV trace length is {trace_length_samples} samples, "
+                        f"but HDF5 stream {point.stream_id} has {native_length}-sample "
+                        "segments. Triggered records cannot span HDF5 segment boundaries."
+                    )
+
+                selection = "random_within_hdf5_segments"
+
+            elif point.storage_format == "zarr":
+                if not reader.is_continuous_stream:
+                    native_length = point.trace_length_samples
+                    if native_length is not None and int(native_length) != trace_length_samples:
+                        raise ValueError(
+                            f"Finite Zarr IV stream {point.stream_id} has native trace "
+                            f"length {native_length}, requested {trace_length_samples}."
+                        )
+                    record_list = None
+                    if nrandoms_iv is not None:
+                        record_list = self._random_native_zarr_records(
+                            point,
+                            int(nrandoms_iv),
+                            seed=self._point_seed(
+                                random_seed, channel, measurement_type, point
+                            ),
+                        )
+                    traces, infos = reader.read_records(
+                        record_list=record_list,
+                        channels=channel,
+                        units="amps",
+                        include_metadata=True,
+                        stack=True,
+                    )
+                    return traces, infos, detector_settings, "native_zarr_traces"
+                selection = "random_across_zarr_stream"
+            else:
+                raise ValueError(
+                    f"Unsupported storage format {point.storage_format!r}."
+                )
+
+            target_nrandoms = nrandoms_iv
+            if target_nrandoms is None:
+                target_nrandoms = self._maximum_nonoverlap_randoms(
+                    point,
+                    trace_length_samples=trace_length_samples,
+                    pretrigger_samples=pretrigger_samples,
+                    min_separation_samples=min_separation_samples,
+                )
+            if target_nrandoms <= 0:
+                raise ValueError(
+                    f"No complete {trace_length_iv_msec:g} ms IV traces fit in "
+                    f"stream {point.stream_id} with the requested edge exclusion."
+                )
+
+            seed = self._point_seed(
+                random_seed, channel, measurement_type, point
+            )
+            randoms = Randoms(
+                point.acquisition_path,
+                streams=point.stream_id,
+                data_type="iv",
+                verbose=False,
+            )
+            random_df = randoms.process(
+                nrandoms=int(target_nrandoms),
+                min_separation_msec=float(min_separation_iv_msec),
+                edge_exclusion_msec=float(edge_exclusion_msec),
+                partition_target_duration_s=None,
+                random_seed=seed,
+                lgc_save=False,
+                lgc_output=True,
+            )
+            record_list = self._random_dataframe_to_record_list(
+                random_df, point.storage_format
+            )
+
+            traces, infos = reader.read_records(
+                record_list=record_list,
+                channels=channel,
+                trace_length_samples=trace_length_samples,
+                pretrigger_length_samples=pretrigger_samples,
+                units="amps",
+                include_metadata=True,
+                stack=True,
+            )
+            return traces, infos, detector_settings, selection
+
+    def _discover_sweep_data(self, data_paths):
+        if self._verbose:
+            print("INFO: Checking sweep data")
+
+        if isinstance(data_paths, (str, Path)):
+            data_paths = [data_paths]
+        else:
+            data_paths = list(data_paths)
+        if not data_paths:
+            raise ValueError("No IV/dIdV acquisition path provided")
+
+        sources = {"iv": None, "didv": None}
+        for path in data_paths:
+            catalog = AcquisitionCatalog(path, verbose=self._verbose)
+            for measurement_type in ("iv", "didv"):
+                view = catalog.filter(measurement_types=measurement_type)
+                if not view.entries:
+                    continue
+                if sources[measurement_type] is not None:
+                    previous = sources[measurement_type]["acquisition_path"]
+                    if Path(previous).resolve() != Path(catalog.acquisition_path).resolve():
+                        raise ValueError(
+                            f"ERROR: {measurement_type} data should be in a single "
+                            "acquisition directory."
+                        )
+                sources[measurement_type] = {
+                    "catalog": catalog,
+                    "view": view,
+                    "acquisition_path": str(catalog.acquisition_path),
+                    "acquisition_name": catalog.acquisition_name,
+                    "base_path": str(catalog.base_path),
+                }
+
+        if sources["iv"] is None and sources["didv"] is None:
+            raise ValueError("No IV or dIdV data were found in the supplied acquisition paths")
+
+        points_by_type = {}
+        for measurement_type in ("iv", "didv"):
+            source = sources[measurement_type]
+            if source is None:
+                points_by_type[measurement_type] = {}
+                continue
+            points_by_type[measurement_type] = self._discover_measurement_points(
+                source["view"], measurement_type
+            )
+
+        channels = sorted(
+            set(points_by_type["iv"]).union(points_by_type["didv"])
+        )
+        output = {}
+        for channel in channels:
+            iv_points = points_by_type["iv"].get(channel, [])
+            didv_points = points_by_type["didv"].get(channel, [])
+            pairs = self._match_iv_didv_points(iv_points, didv_points)
+            output[channel] = {
+                "IV": iv_points or None,
+                "dIdV": didv_points or None,
+                "IV_dIdV": pairs or None,
+            }
+
+        iv_source = sources["iv"]
+        didv_source = sources["didv"]
+        return {
+            "data": output,
+            "acquisition_name_iv": None if iv_source is None else iv_source["acquisition_name"],
+            "acquisition_name_didv": None if didv_source is None else didv_source["acquisition_name"],
+            "base_path_iv": None if iv_source is None else iv_source["base_path"],
+            "base_path_didv": None if didv_source is None else didv_source["base_path"],
+        }
+
+    def _discover_measurement_points(self, catalog_view, measurement_type):
+        entries_by_stream = {}
+        for entry in catalog_view.entries:
+            stream_id = self._entry_stream_id(entry)
+            entries_by_stream.setdefault(stream_id, []).append(entry)
+
+        stream_info = []
+        explicit_scan_channels = set()
+
+        for stream_id, entries in sorted(
+            entries_by_stream.items(), key=lambda item: self._stream_sort_key(item[1][0])
+        ):
+            entries = sorted(entries, key=self._resource_sort_key)
+            first_entry = entries[0]
+            resource = catalog_view.resource_path(first_entry)
+
+            # StreamReader provides the common HDF5/Zarr metadata interface.
+            # ``resource`` is the selected stream resource from the catalog.
+            with StreamReader(resource, verbose=False) as reader:
+                detector_settings = reader.get_detector_settings()
+                metadata = reader.get_metadata()
+
+            scan = metadata.get("scan") or first_entry.get("scan")
+            explicit_scan_channels.update(self._tes_bias_scan_channels(scan))
+
+            record_channels = set()
+            for entry in entries:
+                channels = entry.get("record_channels") or []
+                if isinstance(channels, str):
+                    channels = [channels]
+                record_channels.update(str(name) for name in channels)
+
+            stream_info.append(
+                {
+                    "stream_id": stream_id,
+                    "entries": entries,
+                    "settings": detector_settings,
+                    "scan": scan,
+                    "record_channels": record_channels,
+                }
+            )
+
+        if explicit_scan_channels:
+            sweep_channels = explicit_scan_channels
+        else:
+            sweep_channels = self._infer_legacy_sweep_channels(stream_info)
+            if self._verbose and sweep_channels:
+                print(
+                    f"INFO: {measurement_type} acquisition has no explicit TES-bias "
+                    "scan-channel metadata; inferred channels from bias changes: "
+                    + ", ".join(sorted(sweep_channels))
+                )
+
+        output = {channel: [] for channel in sweep_channels}
+
+        for info in stream_info:
+            entries = info["entries"]
+            settings = info["settings"]
+            first_entry = entries[0]
+            scan = info["scan"]
+            scan_index = self._scan_index(scan)
+
+            for channel in sweep_channels:
+                record_channels = info.get("record_channels") or set()
+                if record_channels and channel not in record_channels:
+                    continue
+                channel_settings = settings.get(channel)
+                if channel_settings is None:
+                    continue
+                bias_amp = self._float_or_nan(channel_settings.get("tes_bias_dc_amps"))
+                if not np.isfinite(bias_amp):
+                    continue
+
+                point = self._make_sweep_point(
+                    catalog_view,
+                    entries,
+                    measurement_type=measurement_type,
+                    stream_id=info["stream_id"],
+                    bias_ua=bias_amp * 1e6,
+                    scan_index=scan_index,
+                )
+                output[channel].append(point)
+
+        for channel in list(output):
+            output[channel] = sorted(output[channel], key=self._point_sort_key)
+            if not output[channel]:
+                output.pop(channel)
+        return output
+
+    def _make_sweep_point(
+        self,
+        catalog_view,
+        entries,
+        *,
+        measurement_type,
+        stream_id,
+        bias_ua,
+        scan_index,
+    ):
+        first = entries[0]
+        storage_format = str(first.get("storage_format") or catalog_view.storage_format)
+        sample_rate = float(first.get("sample_rate_hz") or catalog_view.sample_rate_hz)
+
+        hdf5_dump_segments = []
+        if storage_format == "hdf5":
+            for entry in entries:
+                dump_num = int(entry.get("dump_num") or 0)
+                n_segments = int(entry.get("n_segments") or 0)
+                segment_length = entry.get("segment_length_samples")
+                if segment_length is None and entry.get("segment_duration_s") is not None:
+                    segment_length = int(round(float(entry["segment_duration_s"]) * sample_rate))
+                if segment_length is None:
+                    continue
+                hdf5_dump_segments.append(
+                    (dump_num, n_segments, int(segment_length))
+                )
+
+        trace_lengths = {
+            int(entry["trace_length_samples"])
+            for entry in entries
+            if entry.get("trace_length_samples") is not None
+        }
+        trace_length_samples = next(iter(trace_lengths)) if len(trace_lengths) == 1 else None
+        n_traces_values = [int(entry.get("n_traces") or 0) for entry in entries]
+        n_traces = sum(n_traces_values) if any(n_traces_values) else None
+
+        n_samples = None
+        if storage_format == "zarr" and str(first.get("raw_shape_model")) == "channel_sample":
+            values = [entry.get("n_samples") for entry in entries if entry.get("n_samples") is not None]
+            if values:
+                # Native Zarr normally has exactly one resource per stream.
+                n_samples = int(sum(int(value) for value in values))
+            else:
+                try:
+                    stream_view = catalog_view.filter(streams=stream_id)
+                    n_samples = int(stream_view.n_samples)
+                except Exception:
+                    n_samples = None
+
+        duration_s = float(sum(self._entry_duration_s(entry) for entry in entries))
+        return _SweepPoint(
+            acquisition_path=str(catalog_view.acquisition_path),
+            acquisition_name=str(catalog_view.acquisition_name),
+            measurement_type=measurement_type,
+            stream_id=str(stream_id),
+            stream_num=self._optional_int(first.get("stream_num")),
+            stream_name=first.get("stream_name"),
+            storage_format=storage_format,
+            sample_rate_hz=sample_rate,
+            bias_ua=float(bias_ua),
+            scan_index=scan_index,
+            sequence_index=self._optional_int(first.get("sequence_index")),
+            sequence_repeat_index=self._optional_int(first.get("sequence_repeat_index")),
+            adc_mode=first.get("adc_mode"),
+            raw_shape_model=first.get("raw_shape_model"),
+            duration_s=duration_s,
+            n_samples=n_samples,
+            hdf5_dump_segments=tuple(hdf5_dump_segments),
+            trace_length_samples=trace_length_samples,
+            n_traces=n_traces,
+        )
+
+    def _match_iv_didv_points(self, iv_points, didv_points):
+        if not iv_points or not didv_points:
+            return []
+
+        pairs = []
+        used_iv = set()
+        used_didv = set()
+
+        # Modern same-acquisition data: scan index + repeat is the strongest
+        # identity. Bias is still retained in the output as a consistency check.
+        for i, iv in enumerate(iv_points):
+            if iv.scan_index is None:
+                continue
+            for j, didv in enumerate(didv_points):
+                if j in used_didv or didv.scan_index is None:
+                    continue
+                if Path(iv.acquisition_path).resolve() != Path(didv.acquisition_path).resolve():
+                    continue
+                if iv.scan_index != didv.scan_index:
+                    continue
+                if (
+                    iv.sequence_repeat_index is not None
+                    and didv.sequence_repeat_index is not None
+                    and iv.sequence_repeat_index != didv.sequence_repeat_index
+                ):
+                    continue
+                pairs.append(
+                    _SweepPair(
+                        iv=iv,
+                        didv=didv,
+                        bias_delta_ua=float(didv.bias_ua - iv.bias_ua),
+                        match_method="scan_index",
+                    )
+                )
+                used_iv.add(i)
+                used_didv.add(j)
+                break
+
+        candidates = []
+        for i, iv in enumerate(iv_points):
+            if i in used_iv:
+                continue
+            for j, didv in enumerate(didv_points):
+                if j in used_didv:
+                    continue
+                if self._biases_match(iv.bias_ua, didv.bias_ua):
+                    candidates.append((abs(iv.bias_ua - didv.bias_ua), i, j))
+
+        for _, i, j in sorted(candidates):
+            if i in used_iv or j in used_didv:
+                continue
+            iv = iv_points[i]
+            didv = didv_points[j]
+            pairs.append(
+                _SweepPair(
+                    iv=iv,
+                    didv=didv,
+                    bias_delta_ua=float(didv.bias_ua - iv.bias_ua),
+                    match_method="bias",
+                )
+            )
+            used_iv.add(i)
+            used_didv.add(j)
+
+        return sorted(pairs, key=lambda pair: self._point_sort_key(pair.iv))
+
+    def _assign_sweep_state(self, dataframe):
+        dataframe["state"] = pd.Series(
+            np.full(len(dataframe), np.nan, dtype=object),
+            index=dataframe.index,
+            dtype=object,
+        )
+        if dataframe.empty:
+            return
+
+        tes_bias = dataframe["tes_bias"].to_numpy()
+        if "offset_iv" in dataframe.columns:
+            offset = dataframe["offset_iv"].to_numpy()
+        elif "offset_didv" in dataframe.columns:
+            offset = dataframe["offset_didv"].to_numpy()
+        else:
+            return
+
+        normal_indices = np.asarray(find_linear_segment(tes_bias, offset), dtype=int)
+        if normal_indices.size:
+            dataframe.loc[normal_indices, "state"] = "normal"
+
+        reversed_bias = tes_bias[::-1].copy()
+        reversed_offset = offset[::-1].copy()
+        sc_reversed = np.asarray(
+            find_linear_segment(reversed_bias, reversed_offset), dtype=int
+        )
+        if sc_reversed.size:
+            sc_indices = len(tes_bias) - sc_reversed - 1
+            dataframe.loc[sc_indices, "state"] = "sc"
+
+    def _maximum_nonoverlap_randoms(
+        self,
+        point,
+        *,
+        trace_length_samples,
+        pretrigger_samples,
+        min_separation_samples,
+    ):
+        edge = int(pretrigger_samples)
+        spacing = max(1, int(min_separation_samples))
+
+        if point.storage_format == "hdf5":
+            total = 0
+            for _, n_segments, segment_length in point.hdf5_dump_segments:
+                usable = int(segment_length) - 2 * edge
+                if usable <= 0:
+                    continue
+                capacity = 1 + (usable - 1) // spacing
+                total += int(n_segments) * capacity
+            return total
+
+        if point.storage_format == "zarr":
+            n_samples = point.n_samples
+            if n_samples is None:
+                n_samples = int(round(point.duration_s * point.sample_rate_hz))
+            usable = int(n_samples) - 2 * edge
+            if usable <= 0:
+                return 0
+            return 1 + (usable - 1) // spacing
+
+        return 0
+
+    @staticmethod
+    def _random_dataframe_to_record_list(dataframe, storage_format):
+        if isinstance(dataframe, pd.DataFrame):
+            pdf = dataframe
+        elif hasattr(dataframe, "to_pandas_df"):
+            pdf = dataframe.to_pandas_df()
+        elif hasattr(dataframe, "to_pandas_dataframe"):
+            pdf = dataframe.to_pandas_dataframe()
+        else:
+            raise TypeError(
+                "Randoms output must be a Vaex or pandas dataframe to build a record list"
+            )
+
+        records = []
+        for row in pdf.to_dict(orient="records"):
+            record = {}
+            if storage_format == "hdf5":
+                global_segment = row.get(
+                    "global_segment_num", row.get("global_segment_number")
+                )
+                if global_segment is None:
+                    raise ValueError("HDF5 random row is missing global segment identity")
+                record["global_segment_num"] = int(global_segment)
+                record["segment_trigger_index"] = int(row["segment_trigger_index"])
+            else:
+                record["stream_trigger_index"] = int(row["stream_trigger_index"])
+            records.append(record)
+        return records
+
+    def _random_native_hdf5_records(self, point, n_records, *, seed):
+        all_records = []
+        for dump_num, n_segments, _ in point.hdf5_dump_segments:
+            for segment_num in range(1, n_segments + 1):
+                all_records.append(
+                    {"global_segment_num": dump_num * 100000 + segment_num}
+                )
+        if n_records > len(all_records):
+            raise ValueError(
+                f"Requested nrandoms_iv={n_records}, but HDF5 stream "
+                f"{point.stream_id} contains only {len(all_records)} native segments."
+            )
+        if n_records == len(all_records):
+            return all_records
+        rng = np.random.default_rng(seed)
+        selected = np.sort(rng.choice(len(all_records), size=n_records, replace=False))
+        return [all_records[int(index)] for index in selected]
+
+    def _random_native_zarr_records(self, point, n_records, *, seed):
+        if point.n_traces is None:
+            raise ValueError(
+                f"Unable to determine number of finite Zarr traces for {point.stream_id}."
+            )
+        if n_records > point.n_traces:
+            raise ValueError(
+                f"Requested nrandoms_iv={n_records}, but finite Zarr stream "
+                f"{point.stream_id} contains only {point.n_traces} traces."
+            )
+        rng = np.random.default_rng(seed)
+        selected = np.sort(rng.choice(point.n_traces, size=n_records, replace=False))
+        return [{"trace_index": int(index)} for index in selected]
+
+    @staticmethod
+    def _split_work_items(work_items, n_workers):
+        chunks = np.array_split(np.asarray(work_items, dtype=object), n_workers)
+        return [list(chunk) for chunk in chunks if len(chunk)]
+
+    def _save_filter_data(self, save_path, *, prefer_iv=True):
+        if prefer_iv and self._base_path_iv is not None:
+            base_path = self._base_path_iv
+            group_name = self._acquisition_name_iv
+        else:
+            base_path = self._base_path_didv or self._base_path_iv
+            group_name = self._acquisition_name_didv or self._acquisition_name_iv
+        if save_path is None:
+            save_path = Path(base_path) / "filterdata"
+            save_path = Path(str(save_path).replace("/raw/filterdata", "/filterdata"))
+        else:
+            save_path = Path(save_path)
+
+        if group_name and group_name not in str(save_path):
+            save_path = save_path / group_name
+        save_path.mkdir(parents=True, exist_ok=True)
+
+        now = datetime.now()
+        timestamp_id = now.strftime("D%Y%m%d_T%H%M%S")
+        if self._processing_label is not None:
+            file_name = save_path / f"{self._processing_label}_{timestamp_id}.hdf5"
+        else:
+            file_name = save_path / f"ivsweep_processing_{timestamp_id}.hdf5"
+        self._filter_data.save_hdf5(str(file_name))
+        return str(file_name)
+
+    def _infer_legacy_sweep_channels(self, stream_info):
+        values = {}
+        for info in stream_info:
+            for channel, settings in info["settings"].items():
+                bias = self._float_or_nan(settings.get("tes_bias_dc_amps"))
+                if np.isfinite(bias):
+                    values.setdefault(channel, []).append(bias)
+
+        sweep_channels = set()
+        for channel, biases in values.items():
+            if len(biases) < 2:
+                continue
+            biases = np.asarray(biases, dtype=float)
+            spread = float(np.ptp(biases))
+            scale = max(float(np.max(np.abs(biases))), 1e-15)
+            if spread > max(1e-15, 1e-3 * scale):
+                sweep_channels.add(channel)
+        return sweep_channels
+
+    @staticmethod
+    def _tes_bias_scan_channels(scan):
+        channels = set()
+        if not isinstance(scan, dict):
+            return channels
+        values = scan.get("values")
+        if not isinstance(values, dict):
+            return channels
+        for entry in values.values():
+            if not isinstance(entry, dict):
+                continue
+            target = str(entry.get("target") or "").strip().lower()
+            if target not in {
+                "tes_bias.dc",
+                "tes_bias_dc",
+                "tes_bias.dc_amps",
+                "tes_bias_dc_amps",
+            }:
+                continue
+            entry_channels = entry.get("channels") or []
+            if isinstance(entry_channels, str):
+                entry_channels = [entry_channels]
+            channels.update(str(channel) for channel in entry_channels)
+        return channels
+
+    @staticmethod
+    def _scan_index(scan):
+        if not isinstance(scan, dict) or scan.get("index") is None:
+            return None
+        try:
+            return int(scan["index"])
+        except (TypeError, ValueError):
+            return None
+
+    def _biases_match(self, first, second):
+        return bool(
+            np.isclose(
+                float(first),
+                float(second),
+                rtol=self._bias_tolerance_percent / 100.0,
+                atol=self._bias_tolerance_ua,
+            )
+        )
+
+    @staticmethod
+    def _point_seed(base_seed, channel, measurement_type, point):
+        if base_seed is None:
+            return None
+        text = "|".join(
+            [
+                str(int(base_seed)),
+                str(channel),
+                str(measurement_type),
+                str(point.acquisition_name),
+                str(point.stream_id),
+            ]
+        )
+        digest = hashlib.sha256(text.encode("utf-8")).digest()
+        return int.from_bytes(digest[:8], "big") % (2**32)
+
+    @staticmethod
+    def _entry_stream_id(entry):
+        value = entry.get("stream_id")
+        if value is not None:
+            return str(value)
+        value = entry.get("stream_name")
+        if value is not None:
+            return str(value)
+        value = entry.get("stream_num")
+        if value is not None:
+            return str(value)
+        raise ValueError(f"Catalog entry is missing stream identity: {entry}")
+
+    @staticmethod
+    def _stream_sort_key(entry):
+        value = entry.get("stream_num")
+        if value is not None:
+            return (0, int(value))
+        return (1, str(entry.get("stream_id") or entry.get("stream_name") or ""))
+
+    @staticmethod
+    def _resource_sort_key(entry):
+        dump = entry.get("dump_num")
+        if dump is not None:
+            return (0, int(dump))
+        return (1, str(entry.get("resource_path") or ""))
+
+    @staticmethod
+    def _point_sort_key(point):
+        return (
+            point.scan_index is None,
+            point.scan_index if point.scan_index is not None else 0,
+            point.sequence_repeat_index if point.sequence_repeat_index is not None else 0,
+            point.stream_num if point.stream_num is not None else 0,
+        )
+
+    @staticmethod
+    def _entry_duration_s(entry):
+        if entry.get("duration_s") is not None:
+            return float(entry["duration_s"])
+        sample_rate = entry.get("sample_rate_hz")
+        if sample_rate is None or float(sample_rate) <= 0:
+            return 0.0
+        if entry.get("n_samples") is not None:
+            return float(entry["n_samples"]) / float(sample_rate)
+        if entry.get("n_segments") is not None and entry.get("segment_length_samples") is not None:
+            return (
+                float(entry["n_segments"])
+                * float(entry["segment_length_samples"])
+                / float(sample_rate)
+            )
+        return 0.0
+
+    @staticmethod
+    def _temperature_value(settings, name):
+        for key in (
+            f"temperature_{name}",
+            f"temperature_{name}_k",
+        ):
+            if key in settings:
+                try:
+                    return float(settings[key])
+                except (TypeError, ValueError):
+                    return np.nan
+        return np.nan
+
+    @staticmethod
+    def _optional_int(value):
+        if value is None:
+            return None
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return None
+
+    @staticmethod
+    def _float_or_nan(value):
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return np.nan
+
+    @staticmethod
+    def _float_or_default(value, default):
+        try:
+            value = float(value)
+        except (TypeError, ValueError):
+            return float(default)
+        return value if np.isfinite(value) else float(default)
+
+    @staticmethod
+    def _finite_float(value, name, channel):
+        try:
+            value = float(value)
+        except (TypeError, ValueError) as exc:
+            raise ValueError(f"Invalid {name} for channel {channel}: {value!r}") from exc
+        if not np.isfinite(value):
+            raise ValueError(f"Invalid {name} for channel {channel}: {value!r}")
+        return value