pytesprocess 0.1.1__py3-none-any.whl

pytesprocess/__init__.py

@@ -0,0 +1,9 @@
+from . import process
+from .process import *
+from . import core
+from .core import *
+from .salting import *
+from .utils import *
+from ._version import __version__
+
+from .config import *

pytesprocess/_version.py

@@ -0,0 +1,2 @@
+# special file for defining the current version of the package
+__version__ = "0.1.1"

pytesprocess/cli/__init__.py

@@ -0,0 +1 @@
+from .main import main

pytesprocess/cli/commands/__init__.py

@@ -0,0 +1,5 @@
+"""Command-specific parser/run helpers for the pytesprocess CLI."""
+
+from . import event, filter, ivsweep
+
+__all__ = ["event", "filter", "ivsweep"]

pytesprocess/cli/commands/event.py

@@ -0,0 +1,66 @@
+"""Default event-processing CLI arguments."""
+
+from pytesprocess.workflows.processing import run_processing_workflow
+
+_PROCESS_STEPS = ("randoms", "salting", "trigger", "feature")
+
+
+def configure(parser):
+    parser.add_argument(
+        "--steps",
+        nargs="+",
+        required=True,
+        metavar="STEP[,STEP...]",
+        help=(
+            "Processing steps to run. Values may be space-separated, "
+            "comma-separated, or mixed (for example '--steps randoms trigger "
+            "feature' or '--steps randoms,trigger,feature'). User order is "
+            "ignored; execution order is always randoms -> salting -> trigger "
+            "-> feature. Valid steps: " + ", ".join(_PROCESS_STEPS) + "."
+        ),
+    )
+
+    # Trigger/feature options
+    parser.add_argument("--ntriggers", type=int, default=-1)
+    parser.add_argument("--nevents", type=int, default=-1)
+    parser.add_argument(
+        "--measurement-type",
+        default="background",
+        choices=["background", "calibration", "threshold"],
+        help=(
+            "Raw measurement type for trigger/feature processing. Threshold "
+            "data may be processed directly by the feature step without a "
+            "trigger dataframe."
+        ),
+    )
+    parser.add_argument(
+        "--trigger-dataframe",
+        help="Existing trigger dataframe group/file for feature processing.",
+    )
+    parser.add_argument(
+        "--salting-dataframe",
+        help=(
+            "Existing salting dataframe to inject during trigger/feature "
+            "processing. Do not combine with the salting generation step."
+        ),
+    )
+    parser.add_argument("--external-features")
+    parser.add_argument(
+        "--partition-duration",
+        type=float,
+        default=10.0,
+        dest="partition_duration",
+    )
+
+    # Random-generation options. They are validated only when randoms is
+    # requested, so the same top-level command works for other step sets.
+    count = parser.add_mutually_exclusive_group(required=False)
+    count.add_argument("--nrandoms", type=int)
+    count.add_argument("--random-rate", type=float)
+    parser.add_argument("--min-separation-msec", type=float, default=0.0)
+    parser.add_argument("--edge-exclusion-msec", type=float, default=0.0)
+    parser.add_argument("--random-seed", type=int)
+
+
+def run(args):
+    return run_processing_workflow(args)

pytesprocess/cli/commands/filter.py

@@ -0,0 +1,17 @@
+from pytesprocess.process import FilterDataProcessing
+
+
+def configure(parser):
+    parser.add_argument("--channels", nargs="+")
+
+
+def run(args):
+    proc = FilterDataProcessing(
+        args.acquisition, streams=args.streams, restricted=args.restricted,
+        config_file=args.config, verbose=not args.quiet,
+    )
+    proc.process(
+        channels=args.channels, processing_label=args.processing_label,
+        lgc_output=False, lgc_save=True,
+        save_file_path=args.output, ncores=args.ncores,
+    )

pytesprocess/cli/commands/ivsweep.py

@@ -0,0 +1,29 @@
+from pytesprocess.process import IVSweepProcessing
+
+
+def configure(parser):
+    parser.add_argument("--channels", nargs="+")
+    parser.add_argument("--iv-only", action="store_true")
+    parser.add_argument("--didv-only", action="store_true")
+    parser.add_argument("--trace-length-msec", type=float, default=100.0)
+    parser.add_argument("--nrandoms-iv", type=int)
+    parser.add_argument("--random-seed", type=int)
+
+
+def run(args):
+    if args.iv_only and args.didv_only:
+        raise ValueError("Use only one of --iv-only or --didv-only")
+    proc = IVSweepProcessing(
+        args.acquisition, processing_label=args.processing_label,
+        verbose=not args.quiet,
+    )
+    proc.process(
+        channels=args.channels,
+        enable_iv=not args.didv_only,
+        enable_didv=not args.iv_only,
+        trace_length_iv_msec=args.trace_length_msec,
+        nrandoms_iv=args.nrandoms_iv,
+        random_seed=args.random_seed,
+        lgc_output=False, lgc_save=True,
+        save_path=args.output, ncores=args.ncores,
+    )

pytesprocess/cli/common.py

@@ -0,0 +1,86 @@
+"""Shared helpers for the pytesprocess command-line interface."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pytesdaqx.io import AcquisitionCatalog
+
+from pytesprocess.config import ProcessingConfig
+
+
+def add_common_acquisition_args(parser, *, config=False, ncores=True):
+    parser.add_argument("acquisition", help="Raw pytesdaqx acquisition path")
+    if config:
+        parser.add_argument("--config", required=True, help="Processing YAML file")
+    parser.add_argument("--streams", nargs="+", help="Optional stream IDs")
+    parser.add_argument("--output", help="Base output directory")
+    parser.add_argument("--processing-label", help="Optional human-readable processing label")
+    if ncores:
+        parser.add_argument("--ncores", type=int, default=1, help="Number of local worker processes")
+    parser.add_argument("--restricted", action="store_true", help="Process restricted background data only")
+    parser.add_argument("--quiet", action="store_true", help="Reduce informational output")
+
+
+def catalog_for(args, measurement_types=None):
+    measurement = measurement_types
+    if isinstance(measurement, (list, tuple, set)):
+        unique = {str(item) for item in measurement}
+        background_only = unique == {"background"}
+    else:
+        background_only = (measurement is None or str(measurement) == "background")
+
+    if args.restricted and not background_only:
+        raise ValueError(
+            "--restricted is only valid for background data. It selects "
+            "restricted background data only, not open + restricted data."
+        )
+
+    catalog = AcquisitionCatalog(args.acquisition, verbose=not args.quiet)
+    return catalog.filter(
+        streams=args.streams,
+        measurement_types=measurement_types,
+        restricted=bool(args.restricted) if background_only else False,
+    )
+
+
+def processing_config(args, *, workflows, measurement_types="background", check_resources=True):
+    catalog = catalog_for(args, measurement_types=measurement_types)
+    config = ProcessingConfig(
+        args.config,
+        catalog.record_channels,
+        sample_rate=catalog.sample_rate_hz,
+        verbose=not args.quiet,
+    )
+    config.validate(workflows, check_resources=check_resources, display=not args.quiet)
+    return config, catalog
+
+
+def default_output_base(acquisition):
+    path = Path(acquisition)
+    parent = path.parent
+    if parent.name == "raw":
+        return str(parent.parent / "processed")
+    return str(parent / "processed")
+
+
+def output_base(args):
+    return args.output or default_output_base(args.acquisition)
+
+
+def trigger_edge_parameters(config, catalog):
+    """Return trigger edge exclusion and legacy-HDF5 livetime."""
+    from pytesprocess.core import FilterData
+    from pytesprocess.utils import get_trigger_template_info
+
+    trigger_config = config.get_config("trigger")
+    filter_file = trigger_config["overall"]["filter_file"]
+    filter_data = FilterData()
+    filter_data.load_hdf5(filter_file, overwrite=True)
+    info = get_trigger_template_info(trigger_config, filter_data)
+    edge_msec = float(info["max_edge_exclusion"])
+    livetime = None
+    if catalog.storage_format == "hdf5":
+        nsegments = sum(int(entry.get("n_segments") or 0) for entry in catalog.entries)
+        livetime = float(catalog.get_duration()) - nsegments * 2.0 * edge_msec * 1e-3
+    return edge_msec, livetime

pytesprocess/cli/main.py

@@ -0,0 +1,81 @@
+"""Installed ``pytesprocess`` command-line interface."""
+
+from __future__ import annotations
+
+import argparse
+import multiprocessing
+import sys
+
+from .common import add_common_acquisition_args
+from .commands import event, filter as filter_command, ivsweep
+
+_SECONDARY_COMMANDS = {
+    "filter": filter_command,
+    "ivsweep": ivsweep,
+}
+
+
+def _event_parser():
+    parser = argparse.ArgumentParser(
+        prog="pytesprocess",
+        description=(
+            "TES event-processing workflow. Provide an acquisition and the "
+            "requested --steps. Separate filter and ivsweep workflows are "
+            "available as 'pytesprocess filter ...' and "
+            "'pytesprocess ivsweep ...'."
+        ),
+    )
+    add_common_acquisition_args(parser, config=True, ncores=True)
+    event.configure(parser)
+    return parser
+
+
+def _secondary_parser(name, module):
+    parser = argparse.ArgumentParser(
+        prog=f"pytesprocess {name}",
+        description=f"pytesprocess {name} workflow",
+    )
+    add_common_acquisition_args(
+        parser,
+        config=(name == "filter"),
+        ncores=True,
+    )
+    module.configure(parser)
+    return parser
+
+
+def build_parser(command=None):
+    """Build one CLI parser.
+
+    ``command=None`` builds the default event-processing parser.  ``filter``
+    and ``ivsweep`` build the two explicit secondary workflow parsers.
+    """
+    if command is None:
+        return _event_parser()
+    if command not in _SECONDARY_COMMANDS:
+        raise ValueError(f"Unknown pytesprocess command: {command}")
+    return _secondary_parser(command, _SECONDARY_COMMANDS[command])
+
+
+def main(argv=None):
+    try:
+        multiprocessing.set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    args_in = list(sys.argv[1:] if argv is None else argv)
+
+    if args_in and args_in[0] in _SECONDARY_COMMANDS:
+        command = args_in.pop(0)
+        module = _SECONDARY_COMMANDS[command]
+        parser = build_parser(command)
+        args = parser.parse_args(args_in)
+        return module.run(args)
+
+    parser = build_parser()
+    args = parser.parse_args(args_in)
+    return event.run(args)
+
+
+if __name__ == "__main__":
+    main()

pytesprocess/config/__init__.py

@@ -0,0 +1,4 @@
+from .manager import ProcessingConfig
+from .warnings import ConfigDeprecationWarning
+
+__all__ = ['ProcessingConfig', 'ConfigDeprecationWarning']

pytesprocess/config/loader.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import copy
+from pathlib import Path
+import warnings
+import yaml
+from yaml.loader import SafeLoader
+
+from .warnings import ConfigDeprecationWarning
+
+
+class UniqueKeyLoader(SafeLoader):
+    def construct_mapping(self, node, deep=False):
+        if not isinstance(node, yaml.MappingNode):
+            raise yaml.constructor.ConstructorError(
+                None, None, 'expected a mapping node, but found %s' % node.id,
+                node.start_mark)
+        mapping = {}
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if key in mapping:
+                raise ValueError(
+                    f'ERROR: Duplicate key "{key}" found in YAML file. '
+                    'Duplicate configuration keys are not allowed.'
+                )
+            mapping[key] = self.construct_object(value_node, deep=deep)
+        return mapping
+
+
+def _read_yaml(path: Path):
+    with path.open('r') as handle:
+        data = yaml.load(handle, Loader=UniqueKeyLoader)
+    if data is None:
+        data = {}
+    if not isinstance(data, dict):
+        raise ValueError(f'ERROR: Top-level YAML content in "{path}" must be a mapping.')
+    return data
+
+
+def _deep_merge(base, override):
+    out = copy.deepcopy(base)
+    for key, value in override.items():
+        if key in out and isinstance(out[key], dict) and isinstance(value, dict):
+            out[key] = _deep_merge(out[key], value)
+        else:
+            out[key] = copy.deepcopy(value)
+    return out
+
+
+def load_processing_yaml(yaml_file):
+    """Load one processing YAML, including relative include files.
+
+    config_version >= 2 uses deep merge with included files as bases and the
+    main file as the final override. Legacy files retain the old include
+    precedence (included top-level keys override the main file) for backward
+    compatibility, but include paths are still resolved relative to the parent.
+    """
+    path = Path(yaml_file).expanduser().resolve()
+    if not path.is_file():
+        raise ValueError(f'ERROR: Configuration file "{yaml_file}" not found!')
+
+    root = _read_yaml(path)
+    version = int(root.get('config_version', 1) or 1)
+    includes = root.pop('include', None)
+    if includes is None:
+        return root, version, path
+    if isinstance(includes, (str, Path)):
+        includes = [includes]
+    if not isinstance(includes, list):
+        raise ValueError('ERROR: "include" must be a filename or list of filenames.')
+
+    include_data = {}
+    for include_name in includes:
+        include_path = Path(include_name).expanduser()
+        if not include_path.is_absolute():
+            include_path = path.parent / include_path
+        child, _, _ = load_processing_yaml(include_path)
+        if version >= 2:
+            include_data = _deep_merge(include_data, child)
+        else:
+            include_data.update(copy.deepcopy(child))
+
+    if version >= 2:
+        merged = _deep_merge(include_data, root)
+    else:
+        warnings.warn(
+            'Legacy YAML include precedence is deprecated. In config_version: 2, '
+            'included files are merged first and the current file overrides them.',
+            ConfigDeprecationWarning,
+            stacklevel=2,
+        )
+        merged = copy.deepcopy(root)
+        merged.update(include_data)
+    return merged, version, path