pytesprocess 0.1.1__py3-none-any.whl

pytesprocess/workflows/processing.py

@@ -0,0 +1,317 @@
+"""High-level event-processing workflow used by the top-level ``pytesprocess`` CLI."""
+
+from __future__ import annotations
+
+from pytesprocess.cli.common import (
+    catalog_for,
+    output_base,
+    trigger_edge_parameters,
+)
+from pytesprocess.config import ProcessingConfig
+from pytesprocess.process import Randoms, TriggerProcessing, FeatureProcessing
+from pytesprocess.workflows.salting import generate_salting_dataframes
+
+
+_CANONICAL_STEP_ORDER = ("randoms", "salting", "trigger", "feature")
+_VALID_STEPS = set(_CANONICAL_STEP_ORDER)
+
+
+def normalize_processing_steps(steps):
+    """Return unique requested steps in the canonical execution order.
+
+    ``--steps`` accepts whitespace-separated values, comma-separated values,
+    or a mixture of both.  For example, these are equivalent::
+
+        --steps randoms trigger feature
+        --steps randoms,trigger,feature
+        --steps randoms,trigger feature
+    """
+    requested = []
+    for value in steps or []:
+        for token in str(value).split(","):
+            name = token.strip().lower()
+            if not name:
+                continue
+            if name not in _VALID_STEPS:
+                raise ValueError(
+                    f'Unknown processing step "{name}". Valid steps are '
+                    f'{list(_CANONICAL_STEP_ORDER)}.'
+                )
+            if name not in requested:
+                requested.append(name)
+    if not requested:
+        raise ValueError("pytesprocess requires at least one --steps value.")
+    ordered = [step for step in _CANONICAL_STEP_ORDER if step in requested]
+    return requested, ordered
+
+
+def _validate_dependencies(args, ordered_steps):
+    steps = set(ordered_steps)
+    measurement = args.measurement_type
+
+    if args.restricted and measurement != "background":
+        raise ValueError(
+            "--restricted is only valid with --measurement-type background "
+            "and selects restricted background data only."
+        )
+
+    if measurement == "threshold":
+        if "trigger" in steps:
+            raise ValueError(
+                "The trigger step is not valid for threshold data: NI DAQ has "
+                "already produced finite triggered records. Use --steps feature."
+            )
+        if "randoms" in steps:
+            raise ValueError(
+                "The randoms step is not supported for threshold data in the "
+                "combined process workflow."
+            )
+        if "salting" in steps:
+            raise ValueError(
+                "Salting generation currently selects continuous background "
+                "data. For threshold feature processing, provide an existing "
+                "--salting-dataframe if salt injection is required."
+            )
+        if args.trigger_dataframe:
+            raise ValueError(
+                "Threshold feature processing uses native finite records and "
+                "must not be given --trigger-dataframe."
+            )
+    else:
+        if "feature" in steps and "trigger" not in steps and not args.trigger_dataframe:
+            raise ValueError(
+                f"Feature processing of continuous {measurement} data requires "
+                "either the trigger step in the same workflow or an existing "
+                "--trigger-dataframe."
+            )
+
+    if "trigger" in steps and args.trigger_dataframe:
+        raise ValueError(
+            "--trigger-dataframe cannot be supplied when the trigger step is "
+            "requested; feature processing will use the newly generated triggers."
+        )
+
+    if "salting" in steps and args.salting_dataframe:
+        raise ValueError(
+            "Choose either the salting generation step or --salting-dataframe, "
+            "not both."
+        )
+
+    if args.salting_dataframe and not ({"trigger", "feature"} & steps):
+        raise ValueError(
+            "--salting-dataframe is only meaningful when trigger and/or feature "
+            "processing is requested."
+        )
+
+    if "salting" in steps and measurement != "background":
+        raise ValueError(
+            "Salting generation currently supports background data only."
+        )
+
+    if "randoms" in steps:
+        if measurement != "background":
+            raise ValueError(
+                "The combined randoms step currently supports background data only."
+            )
+        if (args.nrandoms is None) == (args.random_rate is None):
+            raise ValueError(
+                "The randoms step requires exactly one of --nrandoms or "
+                "--random-rate."
+            )
+
+
+def _build_config(args, ordered_steps):
+    """Build and validate all requested YAML-backed workflows up front."""
+    yaml_workflows = [
+        step for step in ordered_steps if step in {"salting", "trigger", "feature"}
+    ]
+    if not yaml_workflows:
+        return None, catalog_for(args, measurement_types=args.measurement_type)
+
+    catalog = catalog_for(args, measurement_types=args.measurement_type)
+    config = ProcessingConfig(
+        args.config,
+        catalog.record_channels,
+        sample_rate=catalog.sample_rate_hz,
+        verbose=not args.quiet,
+    )
+    config.validate(
+        yaml_workflows,
+        check_resources=True,
+        display=not args.quiet,
+    )
+
+    # Salt generation may need trigger-template information for edge exclusion
+    # even when trigger processing itself is not requested. Validate that input
+    # before any generation starts.
+    if "salting" in ordered_steps:
+        salting = config.get_config("salting")
+        if not bool(salting.get("overall", {}).get("do_salt_deadtime", False)):
+            if "trigger" not in yaml_workflows:
+                config.validate(
+                    ["trigger"],
+                    check_resources=True,
+                    display=not args.quiet,
+                )
+
+    return config, catalog
+
+
+def _run_randoms(args):
+    proc = Randoms(
+        args.acquisition,
+        streams=args.streams,
+        processing_label=args.processing_label,
+        data_type="background",
+        restricted=args.restricted,
+        verbose=not args.quiet,
+    )
+    proc.process(
+        nrandoms=args.nrandoms,
+        random_rate=args.random_rate,
+        min_separation_msec=args.min_separation_msec,
+        edge_exclusion_msec=args.edge_exclusion_msec,
+        random_seed=args.random_seed,
+        lgc_save=True,
+        lgc_output=False,
+        save_path=output_base(args),
+    )
+    return proc.get_output_path()
+
+
+def _run_trigger(args, config, catalog, salting_dataframe=None):
+    edge_msec, livetime = trigger_edge_parameters(config, catalog)
+    proc = TriggerProcessing(
+        args.acquisition,
+        config,
+        streams=args.streams,
+        processing_label=args.processing_label,
+        restricted=args.restricted,
+        data_type=args.measurement_type,
+        salting_dataframe=salting_dataframe,
+        verbose=not args.quiet,
+    )
+    proc.process(
+        ntriggers=args.ntriggers,
+        lgc_save=True,
+        lgc_output=False,
+        save_path=output_base(args),
+        ncores=args.ncores,
+        edge_exclusion_msec=edge_msec,
+        livetime=livetime,
+        partition_target_duration_s=args.partition_duration,
+    )
+    return proc.get_output_path()
+
+
+def _run_feature(args, config, trigger_dataframe=None, salting_dataframe=None):
+    proc = FeatureProcessing(
+        args.acquisition,
+        config,
+        streams=args.streams,
+        trigger_dataframe_path=trigger_dataframe,
+        external_file=args.external_features,
+        processing_label=args.processing_label,
+        restricted=args.restricted,
+        data_type=args.measurement_type,
+        salting_dataframe=salting_dataframe,
+        verbose=not args.quiet,
+    )
+    proc.process(
+        nevents=args.nevents,
+        lgc_save=True,
+        lgc_output=False,
+        save_path=output_base(args),
+        ncores=args.ncores,
+    )
+    return proc.get_output_path() if hasattr(proc, "get_output_path") else None
+
+
+def run_processing_workflow(args):
+    requested, ordered = normalize_processing_steps(args.steps)
+    _validate_dependencies(args, ordered)
+
+    if not args.quiet:
+        print("Requested steps: " + ", ".join(requested))
+        print("Execution order: " + " -> ".join(ordered))
+
+    config, catalog = _build_config(args, ordered)
+
+    results = {
+        "steps": ordered,
+        "randoms": None,
+        "salting": [],
+        "trigger": [],
+        "feature": [],
+    }
+
+    if "randoms" in ordered:
+        results["randoms"] = _run_randoms(args)
+
+    salting_paths = []
+    if "salting" in ordered:
+        salting_paths = generate_salting_dataframes(
+            args.acquisition,
+            args.config,
+            streams=args.streams,
+            output=args.output,
+            processing_label=args.processing_label,
+            restricted=args.restricted,
+            verbose=not args.quiet,
+        )
+        results["salting"] = list(salting_paths)
+    elif args.salting_dataframe:
+        salting_paths = [args.salting_dataframe]
+
+    # One generated salting dataframe represents one independent injected-data
+    # workflow (for example one fixed energy). Without salting there is one
+    # normal trigger/feature workflow.
+    injection_contexts = salting_paths if salting_paths else [None]
+
+    if "trigger" in ordered:
+        for salt_path in injection_contexts:
+            trigger_path = _run_trigger(
+                args, config, catalog, salting_dataframe=salt_path
+            )
+            results["trigger"].append(trigger_path)
+
+    if "feature" in ordered:
+        if "trigger" in ordered:
+            trigger_paths = results["trigger"]
+            if len(trigger_paths) != len(injection_contexts):
+                raise RuntimeError(
+                    "Internal trigger/salting workflow count mismatch."
+                )
+            for trigger_path, salt_path in zip(trigger_paths, injection_contexts):
+                results["feature"].append(
+                    _run_feature(
+                        args,
+                        config,
+                        trigger_dataframe=trigger_path,
+                        salting_dataframe=salt_path,
+                    )
+                )
+        else:
+            # Continuous feature-only processing uses the user-provided trigger
+            # dataframe. Threshold processing intentionally uses None so
+            # FeatureProcessing reads native finite records directly.
+            trigger_path = (
+                None if args.measurement_type == "threshold"
+                else args.trigger_dataframe
+            )
+            for salt_path in injection_contexts:
+                results["feature"].append(
+                    _run_feature(
+                        args,
+                        config,
+                        trigger_dataframe=trigger_path,
+                        salting_dataframe=salt_path,
+                    )
+                )
+
+    return results
+
+
+# Backward source-level alias for the short-lived v12 implementation.
+def run_trigger_feature(args):
+    return run_processing_workflow(args)

pytesprocess/workflows/salting.py

@@ -0,0 +1,133 @@
+"""Salt-metadata generation workflow used by the CLI and notebooks."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import vaex as vx
+
+from pytesprocess.config import ProcessingConfig
+from pytesprocess.core import FilterData
+from pytesprocess.salting import SaltGenerator
+from pytesprocess.utils import (
+    get_trigger_template_info, create_dataframe_group,
+    add_dataframe_group_columns,
+)
+from pytesdaqx.io import AcquisitionCatalog
+from qetpy.utils import convert_channel_name_to_list
+
+
+def generate_salting_dataframes(
+    acquisition, config_file, *, streams=None, output=None,
+    processing_label=None, restricted=False, verbose=True,
+):
+    catalog = AcquisitionCatalog(acquisition, verbose=verbose).filter(
+        streams=streams, measurement_types="background", restricted=restricted
+    )
+    config = ProcessingConfig(
+        config_file, catalog.record_channels,
+        sample_rate=catalog.sample_rate_hz, verbose=verbose,
+    )
+    config.validate(["salting"], check_resources=True, display=verbose)
+    salting_config = config.get_config("salting")
+    overall = salting_config["overall"]
+
+    filter_file = overall["filter_file"]
+    didv_file = overall.get("didv_file")
+    do_salt_deadtime = bool(overall.get("do_salt_deadtime", False))
+
+    template_info = {}
+    if not do_salt_deadtime:
+        # Trigger templates define the edge exclusion used to avoid placing
+        # salts where trigger processing cannot recover a full pulse.
+        trigger_config = config.get_config("trigger")
+        if not trigger_config.get("channels"):
+            raise ValueError(
+                "Salting with do_salt_deadtime=false requires trigger configuration "
+                "so the pulse-edge exclusion can be determined."
+            )
+        fd = FilterData()
+        fd.load_hdf5(filter_file, overwrite=True)
+        template_info = get_trigger_template_info(trigger_config, fd)
+
+    filter_data = FilterData(verbose=verbose)
+    filter_data.load_hdf5(filter_file, overwrite=False)
+    if didv_file is not None:
+        filter_data.load_hdf5(didv_file, overwrite=False)
+
+    generator = SaltGenerator(filter_data, verbose=verbose)
+    generator.set_raw_data(acquisition, streams=streams, restricted=restricted)
+
+    energies = overall.get("energies")
+    pdf_file = overall.get("dm_pdf_file")
+    if energies is None:
+        energies = [None]
+    elif not isinstance(energies, list):
+        energies = [energies]
+    if pdf_file is not None and energies != [None]:
+        raise ValueError("Use fixed energies or dm_pdf_file, not both")
+
+    nsalt = int(overall.get("nsalt", 100))
+    random_seed = overall.get("random_seed")
+    coincident = bool(overall.get("coincident_salts", False))
+
+    if output:
+        base = Path(output)
+    else:
+        parent = Path(acquisition).parent
+        base = (parent.parent / "processed") if parent.name == "raw" else (parent / "processed")
+    if catalog.acquisition_name not in str(base):
+        base = base / catalog.acquisition_name
+    facility = catalog.entries[0].get("facility")
+    results = []
+
+    for energy_index, energy in enumerate(energies):
+        channel_frames = []
+        shared_positions = None
+        for channel_index, (channel, chan_config) in enumerate(
+            salting_config["channels"].items()
+        ):
+            seed = random_seed
+            if seed is not None and not coincident:
+                seed = int(seed) + channel_index
+
+            chan_list = convert_channel_name_to_list(channel)
+            efficiency = chan_config.get("collection_efficiency", 1.0)
+            if len(chan_list) > 1 and not isinstance(efficiency, (list, tuple)):
+                efficiency = [efficiency] * len(chan_list)
+
+            edge_exclusion_msec = 0.0
+            if not do_salt_deadtime:
+                edge_exclusion_msec = float(
+                    template_info.get("max_edge_exclusion", 0.0)
+                )
+
+            dataframe = generator.generate_metadata(
+                channel,
+                template_tag=chan_config["template_tag"],
+                dpdi_tag=chan_config.get("dpdi_tag"),
+                dpdi_poles=chan_config.get("dpdi_poles"),
+                collection_efficiency=efficiency,
+                energy_eV=None if energy is None else float(energy),
+                dm_pdf_file=pdf_file,
+                nsalt=nsalt,
+                positions=shared_positions if coincident else None,
+                edge_exclusion_msec=edge_exclusion_msec,
+                random_seed=seed,
+            )
+            if coincident and shared_positions is None:
+                shared_positions = generator.get_positions().copy()
+            channel_frames.append(dataframe)
+
+        final = channel_frames[0] if len(channel_frames) == 1 else vx.concat(channel_frames)
+        descriptor = "salting_dm" if energy is None else f"salting_{float(energy):g}eV"
+        group = create_dataframe_group(
+            base, dataframe_type="salting", facility=facility,
+            name_prefix=descriptor, processing_label=processing_label,
+            restricted=restricted,
+        )
+        add_dataframe_group_columns(final, group, 1)
+        file_name = group.file_path(1)
+        final.export_hdf5(file_name, mode="w")
+        results.append(file_name)
+
+    return results

pytesprocess-0.1.1.dist-info/METADATA

@@ -0,0 +1,211 @@
+Metadata-Version: 2.4
+Name: pytesprocess
+Version: 0.1.1
+Summary: Detector data processing for pytesdaqx acquisitions
+Author-email: Bruno Serfass <serfass@berkeley.edu>, Samuel Watkins <samwatkins@berkeley.edu>
+License: MIT License
+        
+        Copyright (c) 2022 Samuel Watkins
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/spice-herald/pytesprocess
+Project-URL: Repository, https://github.com/spice-herald/pytesprocess
+Keywords: detector,TES,DAQ,signal processing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: scipy
+Requires-Dist: matplotlib
+Requires-Dist: PyYAML
+Requires-Dist: qetpy>=1.8.6
+Requires-Dist: pandas
+Requires-Dist: pytesdaqx
+Requires-Dist: humanfriendly
+Requires-Dist: vaex
+Requires-Dist: pyarrow
+Requires-Dist: lmfit
+Requires-Dist: cloudpickle
+Provides-Extra: dev
+Requires-Dist: build; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# pytesprocess
+
+`pytesprocess` is the detector-processing layer used with raw acquisitions written
+by `pytesdaqx`. It provides random-event selection, software triggering,
+feature extraction, salting, IV/dIdV processing, noise/filter generation, and
+persistence of processed event products as Vaex HDF5 dataframes.
+
+The current code targets Python 3.11+ and the modern `pytesdaqx` acquisition
+model (`acquisition` + `stream`).
+
+## Installation
+
+For a development checkout:
+
+```bash
+pip install -e .
+```
+
+Runtime dependencies and the installed CLI entry point are defined entirely in
+`pyproject.toml`.
+
+## Command-line interface
+
+Installing the package provides:
+
+```bash
+pytesprocess --help
+```
+
+The normal event-processing workflow is selected directly from the top-level
+command:
+
+```bash
+pytesprocess ACQUISITION \
+    --config processing.yaml \
+    --steps randoms trigger feature
+```
+
+`--steps` is mandatory. It accepts either whitespace-separated or
+comma-separated values, including mixed input:
+
+```bash
+--steps randoms trigger feature
+--steps randoms,trigger,feature
+--steps randoms,trigger feature
+```
+
+All are equivalent. The execution order is normalized internally to:
+
+```text
+randoms -> salting -> trigger -> feature
+```
+
+The separate detector-characterization/filter workflows remain explicit:
+
+```bash
+pytesprocess filter ACQUISITION --config processing.yaml
+pytesprocess ivsweep ACQUISITION
+```
+
+Example:
+
+```bash
+pytesprocess acquisition_I2_D20260807_T170432.zarr \
+    --config continuous_data_processing_v2.yaml \
+    --steps randoms,trigger,feature \
+    --nrandoms 300 \
+    --nevents 300 \
+    --ncores 4
+```
+
+See [docs/user/cli.md](docs/user/cli.md) for the current CLI behavior.
+
+## Configuration
+
+New processing configuration uses `config_version: 2` and separates workflow
+sections explicitly:
+
+```yaml
+config_version: 2
+
+resources:
+  filter_file: /path/to/filterdata.hdf5
+
+trigger:
+  global: {}
+  channels: {}
+
+feature:
+  global:
+    trace_length_msec: 20
+    pretrigger_length_msec: 10
+  presets: {}
+  channels: {}
+
+salting:
+  global: {}
+  channels: {}
+
+filter: {}
+```
+
+Feature trace lengths are specified in milliseconds in v2 configuration. The
+resolver converts them to samples using the selected acquisition sample rate.
+Channel selectors include `all`, shell-style globs such as `Z1P*`, comma groups
+for applying one block to several independent channels, and explicit
+multi-channel expressions such as `A|B`.
+
+See [docs/user/configuration.md](docs/user/configuration.md).
+
+## Processed dataframe identity
+
+Each Vaex processing product has one dataframe-group identity:
+
+```text
+dataframe_group_name
+dataframe_group_id
+dataframe_group_number
+dataframe_file_index
+processing_label
+```
+
+For example:
+
+```text
+trigger_I2_D20260908_T123456/
+  trigger_I2_D20260908_T123456_F0001.hdf5
+  trigger_I2_D20260908_T123456_F0002.hdf5
+```
+
+`F####` is a processing task/output shard index, not a second stream or series
+identifier. Raw provenance uses canonical stream fields such as `stream_id`,
+`stream_number`, and `stream_trigger_index`.
+
+See [docs/user/outputs.md](docs/user/outputs.md).
+
+## Package structure
+
+The current package layout intentionally separates reusable analysis objects
+from acquisition-processing workflows:
+
+```text
+pytesprocess/
+  core/       reusable analysis/data objects and algorithms
+  process/    raw/dataframe processing executors
+  config/     YAML loading, selection, resolution, validation
+  salting/    salt metadata generation and waveform injection
+  workflows/  multi-step orchestration used by the CLI
+  cli/        command-line parsing and dispatch
+  utils/      shared utilities and HDF5/dataframe helpers
+```
+
+See [docs/developer/architecture.md](docs/developer/architecture.md) for the
+current developer-oriented architecture.
+
+## Notes on multiprocessing
+
+The existing Vaex/PyArrow and numerical-library thread limits are intentional.
+They were introduced to avoid thread oversubscription and unstable/slower
+multicore processing. They should not be removed or relocated without dedicated
+multicore benchmarking.