pytesprocess 0.1.1__tar.gz → 0.1.2__tar.gz

{pytesprocess-0.1.1/pytesprocess.egg-info → pytesprocess-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytesprocess
-Version: 0.1.1
+Version: 0.1.2
 Summary: Detector data processing for pytesdaqx acquisitions
 Author-email: Bruno Serfass <serfass@berkeley.edu>, Samuel Watkins <samwatkins@berkeley.edu>
 License: MIT License
@@ -37,6 +37,7 @@ Requires-Dist: matplotlib
 Requires-Dist: PyYAML
 Requires-Dist: qetpy>=1.8.6
 Requires-Dist: pandas
+Requires-Dist: tables
 Requires-Dist: pytesdaqx
 Requires-Dist: humanfriendly
 Requires-Dist: vaex
@@ -106,6 +107,7 @@ The separate detector-characterization/filter workflows remain explicit:
 ```bash
 pytesprocess filter ACQUISITION --config processing.yaml
 pytesprocess ivsweep ACQUISITION
+pytesprocess vibration ACQUISITION --config vibration.yaml
 ```
 
 Example:

{pytesprocess-0.1.1 → pytesprocess-0.1.2}/README.md

@@ -56,6 +56,7 @@ The separate detector-characterization/filter workflows remain explicit:
 ```bash
 pytesprocess filter ACQUISITION --config processing.yaml
 pytesprocess ivsweep ACQUISITION
+pytesprocess vibration ACQUISITION --config vibration.yaml
 ```
 
 Example:

{pytesprocess-0.1.1 → pytesprocess-0.1.2}/pyproject.toml

@@ -21,6 +21,7 @@ dependencies = [
   "PyYAML",
   "qetpy>=1.8.6",
   "pandas",
+  "tables",
   "pytesdaqx",
   "humanfriendly",
   "vaex",

{pytesprocess-0.1.1 → pytesprocess-0.1.2}/pytesprocess/_version.py

@@ -1,2 +1,2 @@
 # special file for defining the current version of the package
-__version__ = "0.1.1"
+__version__ = "0.1.2"

pytesprocess-0.1.2/pytesprocess/cli/__init__.py

@@ -0,0 +1,16 @@
+"""Command-line interface for pytesprocess.
+
+``main`` is re-exported lazily: ``pytesprocess.workflows`` imports helpers from
+``pytesprocess.cli.common``, so eagerly pulling in the command modules here
+would make that a circular import.
+"""
+
+__all__ = ["main"]
+
+
+def __getattr__(name):
+    if name == "main":
+        from .main import main
+
+        return main
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

pytesprocess-0.1.2/pytesprocess/cli/commands/__init__.py

@@ -0,0 +1,5 @@
+"""Command-specific parser/run helpers for the pytesprocess CLI."""
+
+from . import event, feature, filter, ivsweep, randoms, trigger, vibration
+
+__all__ = ["event", "feature", "filter", "ivsweep", "randoms", "trigger", "vibration"]

{pytesprocess-0.1.1 → pytesprocess-0.1.2}/pytesprocess/cli/commands/event.py

@@ -51,6 +51,17 @@ def configure(parser):
         default=10.0,
         dest="partition_duration",
     )
+    parser.add_argument(
+        "--memory-limit",
+        default="2GB",
+        help=(
+            "Memory safety limit per worker, not shared between them. For "
+            "continuous triggering it bounds the retained waveforms of one "
+            "connected pileup/coincidence group, whose size is set by the "
+            "data rather than by how much is processed at a time; raise it "
+            "if triggering reports the buffer limit."
+        ),
+    )
 
     # Random-generation options. They are validated only when randoms is
     # requested, so the same top-level command works for other step sets.

pytesprocess-0.1.2/pytesprocess/cli/commands/feature.py

@@ -0,0 +1,144 @@
+"""Feature work-unit planning and execution.
+
+Two commands that between them let feature processing be divided across any
+number of workers: ``plan`` writes the division once, ``run`` executes one
+piece of it.  A unit is a plain command line, so the same division can be
+driven by a shell loop, a job array, or anything else.
+"""
+
+from pathlib import Path
+
+from pytesprocess.cli.common import add_common_acquisition_args, output_base
+from pytesprocess.process import FeatureProcessing
+from pytesprocess.process.feature_work import load_plan
+
+
+# This command builds its own sub-parsers, so the shared acquisition
+# arguments are added per action rather than once at the top level.
+COMMON_ARGS = False
+
+_MEASUREMENT_TYPES = ["background", "calibration"]
+
+
+def _add_shared(parser):
+    add_common_acquisition_args(parser, config=True, ncores=False)
+    parser.add_argument(
+        "--measurement-type",
+        default="background",
+        choices=_MEASUREMENT_TYPES,
+        help="Raw measurement type of the data behind the trigger dataframe.",
+    )
+    parser.add_argument("--external-features")
+    parser.add_argument(
+        "--salting-dataframe",
+        help="Existing salting dataframe to inject during feature processing.",
+    )
+
+
+def configure(parser):
+    actions = parser.add_subparsers(dest="action", required=True)
+
+    plan = actions.add_parser(
+        "plan",
+        help="divide a trigger dataframe into feature work units",
+    )
+    _add_shared(plan)
+    plan.add_argument(
+        "--trigger-dataframe",
+        required=True,
+        help="Trigger dataframe group or shard(s) to divide.",
+    )
+    size = plan.add_mutually_exclusive_group()
+    size.add_argument(
+        "--units",
+        type=int,
+        help="Number of work units. Sizes then differ by at most one row.",
+    )
+    size.add_argument(
+        "--rows-per-unit",
+        type=int,
+        help="Fixed rows per unit, which bounds the memory one worker needs.",
+    )
+    plan.add_argument(
+        "--plan-file",
+        help="Where to write the plan. Default: plan.json in the new group.",
+    )
+    plan.add_argument("--campaign", help="Campaign name, recorded for provenance.")
+
+    run = actions.add_parser(
+        "run",
+        help="process exactly one work unit of a plan",
+    )
+    _add_shared(run)
+    run.add_argument("--plan", required=True, help="Plan file to read.")
+    run.add_argument(
+        "--unit", type=int, required=True, help="1-based unit index to process."
+    )
+    run.add_argument(
+        "--trigger-dataframe",
+        help="Trigger dataframe group. Default: derived from the plan itself.",
+    )
+    run.add_argument(
+        "--product-group",
+        help="Override the product group recorded in the plan.",
+    )
+    run.add_argument("--memory-limit", default="2GB")
+
+
+def run(args):
+    if args.action == "plan":
+        return _plan(args)
+    return _run_unit(args)
+
+
+def _processor(args, trigger_dataframe):
+    return FeatureProcessing(
+        args.acquisition,
+        args.config,
+        streams=args.streams,
+        trigger_dataframe_path=trigger_dataframe,
+        external_file=args.external_features,
+        processing_label=args.processing_label,
+        restricted=args.restricted,
+        data_type=args.measurement_type,
+        salting_dataframe=args.salting_dataframe,
+        verbose=not args.quiet,
+    )
+
+
+def _plan(args):
+    proc = _processor(args, args.trigger_dataframe)
+    return proc.plan(
+        n_units=args.units,
+        rows_per_unit=args.rows_per_unit,
+        save_path=output_base(args),
+        plan_path=args.plan_file,
+        campaign=args.campaign,
+    )
+
+
+def _run_unit(args):
+    plan = load_plan(args.plan)
+    trigger_dataframe = args.trigger_dataframe or _trigger_path_from_plan(plan)
+    proc = _processor(args, trigger_dataframe)
+    return proc.process_unit(
+        plan,
+        args.unit,
+        product_group=args.product_group,
+        memory_limit=args.memory_limit,
+    )
+
+
+def _trigger_path_from_plan(plan):
+    """Locate the input trigger dataframe from the plan's own segments.
+
+    A unit job is told a plan and an index; it should not also have to be
+    told where the input lives, since the plan already says.
+    """
+    for unit in plan.get("units", []):
+        for segment in unit.get("selector", {}).get("segments", []):
+            return str(Path(segment["file"]).parent)
+    raise ValueError(
+        "ERROR: This plan has no row segments, so the input trigger "
+        "dataframe cannot be located. Pass --trigger-dataframe explicitly."
+    )

pytesprocess-0.1.2/pytesprocess/cli/commands/randoms.py

@@ -0,0 +1,88 @@
+"""Random-trigger work-unit planning and execution.
+
+``plan`` describes the selection as one unit and records the seed; ``run``
+generates it.  The step is one unit by design: randoms are drawn across the
+whole selection at once, so dividing it would change the product rather than
+distribute the work.
+"""
+
+from pytesprocess.cli.common import add_common_acquisition_args, output_base
+from pytesprocess.process import Randoms
+from pytesprocess.process.work_plan import load_plan
+
+
+# This command builds its own sub-parsers, so the shared acquisition
+# arguments are added per action rather than once at the top level.
+COMMON_ARGS = False
+
+
+def _add_shared(parser):
+    add_common_acquisition_args(parser, config=False, ncores=False)
+
+
+def configure(parser):
+    actions = parser.add_subparsers(dest="action", required=True)
+
+    plan = actions.add_parser(
+        "plan", help="describe the random selection as a work unit"
+    )
+    _add_shared(plan)
+    count = plan.add_mutually_exclusive_group(required=True)
+    count.add_argument("--nrandoms", type=int, help="Exact number to generate.")
+    count.add_argument(
+        "--random-rate", type=float,
+        help="Average rate in Hz; the count follows from the live time.",
+    )
+    plan.add_argument("--min-separation-msec", type=float, default=0.0)
+    plan.add_argument("--edge-exclusion-msec", type=float, default=0.0)
+    plan.add_argument("--partition-duration", type=float, dest="partition_duration")
+    plan.add_argument(
+        "--random-seed", type=int,
+        help="Seed to record. One is generated and recorded if omitted.",
+    )
+    plan.add_argument("--plan-file")
+    plan.add_argument("--campaign")
+
+    run = actions.add_parser("run", help="generate the randoms of a plan")
+    _add_shared(run)
+    run.add_argument("--plan", required=True)
+    run.add_argument("--unit", type=int, default=1)
+    run.add_argument("--product-group")
+
+
+def run(args):
+    if args.action == "plan":
+        return _plan(args)
+    return _run_unit(args)
+
+
+def _processor(args):
+    return Randoms(
+        args.acquisition,
+        streams=args.streams,
+        processing_label=args.processing_label,
+        data_type="background",
+        restricted=args.restricted,
+        verbose=not args.quiet,
+    )
+
+
+def _plan(args):
+    return _processor(args).plan(
+        nrandoms=args.nrandoms,
+        random_rate=args.random_rate,
+        min_separation_msec=args.min_separation_msec,
+        edge_exclusion_msec=args.edge_exclusion_msec,
+        partition_target_duration_s=args.partition_duration,
+        random_seed=args.random_seed,
+        save_path=output_base(args),
+        plan_path=args.plan_file,
+        campaign=args.campaign,
+    )
+
+
+def _run_unit(args):
+    plan = load_plan(args.plan, step="randoms")
+    return _processor(args).process_unit(
+        plan, args.unit, product_group=args.product_group
+    )

pytesprocess-0.1.2/pytesprocess/cli/commands/trigger.py

@@ -0,0 +1,118 @@
+"""Trigger work-unit planning and execution.
+
+``plan`` divides the selected streams into units, one per stream, and records
+the live time and edge exclusion once so that every unit stamps the same
+values.  ``run`` executes one unit of that plan.
+"""
+
+from pytesprocess.cli.common import (
+    add_common_acquisition_args,
+    output_base,
+    processing_config,
+    trigger_edge_parameters,
+)
+from pytesprocess.process import TriggerProcessing
+from pytesprocess.process.work_plan import load_plan
+
+
+# This command builds its own sub-parsers, so the shared acquisition
+# arguments are added per action rather than once at the top level.
+COMMON_ARGS = False
+
+_MEASUREMENT_TYPES = ["background", "calibration"]
+
+
+def _add_shared(parser):
+    add_common_acquisition_args(parser, config=True, ncores=False)
+    parser.add_argument(
+        "--measurement-type",
+        default="background",
+        choices=_MEASUREMENT_TYPES,
+    )
+    parser.add_argument(
+        "--salting-dataframe",
+        help="Existing salting dataframe to inject while triggering.",
+    )
+    parser.add_argument(
+        "--partition-duration", type=float, default=10.0,
+        dest="partition_duration",
+    )
+
+
+def configure(parser):
+    actions = parser.add_subparsers(dest="action", required=True)
+
+    plan = actions.add_parser(
+        "plan", help="divide the selected streams into trigger work units"
+    )
+    _add_shared(plan)
+    plan.add_argument(
+        "--plan-file",
+        help="Where to write the plan. Default: plan.json in the new group.",
+    )
+    plan.add_argument("--campaign", help="Campaign name, recorded for provenance.")
+
+    run = actions.add_parser("run", help="process exactly one work unit of a plan")
+    _add_shared(run)
+    run.add_argument("--plan", required=True, help="Plan file to read.")
+    run.add_argument(
+        "--unit", type=int, required=True, help="1-based unit index to process."
+    )
+    run.add_argument(
+        "--product-group", help="Override the product group recorded in the plan."
+    )
+    run.add_argument("--ntriggers", type=int, default=-1)
+    run.add_argument("--memory-limit", default="2GB")
+
+
+def run(args):
+    if args.action == "plan":
+        return _plan(args)
+    return _run_unit(args)
+
+
+def _processor(args, config):
+    return TriggerProcessing(
+        args.acquisition,
+        config,
+        streams=args.streams,
+        processing_label=args.processing_label,
+        restricted=args.restricted,
+        data_type=args.measurement_type,
+        salting_dataframe=args.salting_dataframe,
+        verbose=not args.quiet,
+    )
+
+
+def _plan(args):
+    config, catalog = processing_config(
+        args, workflows=["trigger"], measurement_types=args.measurement_type
+    )
+    # Computed once, over the whole selection, and carried in the plan: live
+    # time is not a property any single unit can work out for itself.
+    edge_msec, livetime = trigger_edge_parameters(config, catalog)
+
+    proc = _processor(args, config)
+    return proc.plan(
+        save_path=output_base(args),
+        plan_path=args.plan_file,
+        campaign=args.campaign,
+        edge_exclusion_msec=edge_msec,
+        livetime=livetime,
+        partition_target_duration_s=args.partition_duration,
+    )
+
+
+def _run_unit(args):
+    plan = load_plan(args.plan, step="trigger")
+    config, _ = processing_config(
+        args, workflows=["trigger"], measurement_types=args.measurement_type
+    )
+    proc = _processor(args, config)
+    return proc.process_unit(
+        plan,
+        args.unit,
+        product_group=args.product_group,
+        ntriggers=args.ntriggers,
+        memory_limit=args.memory_limit,
+    )

pytesprocess-0.1.2/pytesprocess/cli/commands/vibration.py

@@ -0,0 +1,18 @@
+from pytesprocess.vibration import VibrationProcessing
+
+
+def configure(parser):
+    parser.add_argument("--config", help="Processing YAML file with a 'vibration' section")
+    parser.add_argument("--channels", nargs="+")
+
+
+def run(args):
+    proc = VibrationProcessing(
+        args.acquisition, config_file=args.config, streams=args.streams,
+        restricted=args.restricted, verbose=not args.quiet,
+    )
+    proc.process(
+        channels=args.channels, ncores=args.ncores,
+        lgc_output=False, lgc_save=True,
+        save_path=args.output, processing_label=args.processing_label,
+    )

{pytesprocess-0.1.1 → pytesprocess-0.1.2}/pytesprocess/cli/main.py

@@ -7,11 +7,18 @@ import multiprocessing
 import sys
 
 from .common import add_common_acquisition_args
-from .commands import event, filter as filter_command, ivsweep
+from .commands import (
+    event, feature, filter as filter_command, ivsweep, randoms, trigger,
+    vibration,
+)
 
 _SECONDARY_COMMANDS = {
+    "feature": feature,
     "filter": filter_command,
     "ivsweep": ivsweep,
+    "randoms": randoms,
+    "trigger": trigger,
+    "vibration": vibration,
 }
 
 
@@ -20,9 +27,9 @@ def _event_parser():
         prog="pytesprocess",
         description=(
             "TES event-processing workflow. Provide an acquisition and the "
-            "requested --steps. Separate filter and ivsweep workflows are "
-            "available as 'pytesprocess filter ...' and "
-            "'pytesprocess ivsweep ...'."
+            "requested --steps. Separate filter, ivsweep and vibration "
+            "workflows are available as 'pytesprocess filter ...', "
+            "'pytesprocess ivsweep ...' and 'pytesprocess vibration ...'."
         ),
     )
     add_common_acquisition_args(parser, config=True, ncores=True)
@@ -35,11 +42,14 @@ def _secondary_parser(name, module):
         prog=f"pytesprocess {name}",
         description=f"pytesprocess {name} workflow",
     )
-    add_common_acquisition_args(
-        parser,
-        config=(name == "filter"),
-        ncores=True,
-    )
+    # A command with its own sub-actions adds the shared acquisition
+    # arguments to each of them instead of once here.
+    if getattr(module, "COMMON_ARGS", True):
+        add_common_acquisition_args(
+            parser,
+            config=(name == "filter"),
+            ncores=True,
+        )
     module.configure(parser)
     return parser
 
@@ -47,8 +57,8 @@ def _secondary_parser(name, module):
 def build_parser(command=None):
     """Build one CLI parser.
 
-    ``command=None`` builds the default event-processing parser.  ``filter``
-    and ``ivsweep`` build the two explicit secondary workflow parsers.
+    ``command=None`` builds the default event-processing parser.  ``filter``,
+    ``ivsweep`` and ``vibration`` build the explicit secondary workflow parsers.
     """
     if command is None:
         return _event_parser()
@@ -58,6 +68,7 @@ def build_parser(command=None):
 
 
 def main(argv=None):
+    """Return zero after a successful workflow; processing errors propagate."""
     try:
         multiprocessing.set_start_method("spawn")
     except RuntimeError:
@@ -70,12 +81,14 @@ def main(argv=None):
         module = _SECONDARY_COMMANDS[command]
         parser = build_parser(command)
         args = parser.parse_args(args_in)
-        return module.run(args)
+        module.run(args)
+        return 0
 
     parser = build_parser()
     args = parser.parse_args(args_in)
-    return event.run(args)
+    event.run(args)
+    return 0
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

{pytesprocess-0.1.1 → pytesprocess-0.1.2}/pytesprocess/config/manager.py

@@ -5,7 +5,10 @@ import warnings
 
 from .loader import load_processing_yaml
 from .warnings import ConfigDeprecationWarning
-from .resolvers import FeatureConfigResolver, TriggerConfigResolver, SaltingConfigResolver
+from .resolvers import (
+    FeatureConfigResolver, TriggerConfigResolver, SaltingConfigResolver,
+    VibrationConfigResolver,
+)
 from .resolvers.common import rename_obsolete_keys
 from .validation import (
     normalize_workflows, validate_resolved_workflow,
@@ -20,7 +23,8 @@ _LEGACY_SALTING_OVERALL = {
 }
 _RESERVED_TOP_LEVEL = {
     'config_version', 'resources', 'trigger', 'feature', 'salting', 'filter',
-    'ivsweep', 'didv', 'noise', 'template', 'global', 'filter_file', 'didv_file'
+    'ivsweep', 'didv', 'noise', 'template', 'vibration', 'global', 'filter_file',
+    'didv_file'
 }
 
 
@@ -73,13 +77,13 @@ class ProcessingConfig:
         Returns
         -------
         dict
-            Validation summary suitable for a future CLI startup report.
+            Validation summary (also displayed at CLI startup).
 
         Notes
         -----
-        This method performs no processing and intentionally is not called by
-        ``scripts/process.py`` yet.  It lets a future multi-step CLI validate
-        all requested workflows before starting the first processing stage.
+        This method performs no processing. The CLI workflows call it to
+        validate all requested workflows before starting the first
+        processing stage.
         """
         names = normalize_workflows(workflows)
         if names is None:
@@ -103,7 +107,7 @@ class ProcessingConfig:
     def _configured_workflows(self):
         names = []
         for name in ('trigger', 'feature', 'salting', 'filter', 'ivsweep',
-                     'didv', 'noise', 'template'):
+                     'didv', 'noise', 'template', 'vibration'):
             section = self._canonical.get(name, {})
             if not isinstance(section, dict):
                 continue
@@ -125,7 +129,7 @@ class ProcessingConfig:
                 'global': copy.deepcopy(self._canonical['resources'])
             }
             for name in ('trigger', 'feature', 'salting', 'filter', 'ivsweep',
-                         'didv', 'noise', 'template'):
+                         'didv', 'noise', 'template', 'vibration'):
                 out[name] = self.get_config(name)
             return out
 
@@ -151,6 +155,10 @@ class ProcessingConfig:
                 self._canonical['salting'], self._canonical['resources'],
                 self._available_channels, version=self._version,
             ).resolve()
+        elif name == 'vibration':
+            resolved = VibrationConfigResolver(
+                self._canonical['vibration'], self._available_channels,
+            ).resolve()
         elif name in {'filter', 'ivsweep'}:
             resolved = copy.deepcopy(self._canonical.get(name, {}))
         elif name in {'didv', 'noise', 'template'}:
@@ -169,7 +177,7 @@ class ProcessingConfig:
     def _normalize_v2(self, raw):
         allowed = {
             'config_version', 'resources', 'trigger', 'feature', 'salting',
-            'filter', 'ivsweep', 'didv', 'noise', 'template'
+            'filter', 'ivsweep', 'didv', 'noise', 'template', 'vibration'
         }
         unknown = set(raw) - allowed
         if unknown:
@@ -190,6 +198,7 @@ class ProcessingConfig:
             'didv': copy.deepcopy(raw.get('didv', {})),
             'noise': copy.deepcopy(raw.get('noise', {})),
             'template': copy.deepcopy(raw.get('template', {})),
+            'vibration': copy.deepcopy(raw.get('vibration', {})),
         }
 
     def _normalize_v2_workflow(self, section, name):
@@ -230,6 +239,7 @@ class ProcessingConfig:
         didv_cfg = work.pop('didv', {}) or {}
         noise_cfg = work.pop('noise', {}) or {}
         template_cfg = work.pop('template', {}) or {}
+        vibration_cfg = work.pop('vibration', {}) or {}
         global_feature = work.pop('global', {}) or {}
         work.pop('config_version', None)
         work.pop('resources', None)
@@ -262,6 +272,7 @@ class ProcessingConfig:
             'didv': didv_cfg,
             'noise': noise_cfg,
             'template': template_cfg,
+            'vibration': vibration_cfg,
         }
 
     @staticmethod

{pytesprocess-0.1.1 → pytesprocess-0.1.2}/pytesprocess/config/resolvers/__init__.py

@@ -1,5 +1,7 @@
 from .feature import FeatureConfigResolver
 from .trigger import TriggerConfigResolver
 from .salting import SaltingConfigResolver
+from .vibration import VibrationConfigResolver
 
-__all__ = ['FeatureConfigResolver', 'TriggerConfigResolver', 'SaltingConfigResolver']
+__all__ = ['FeatureConfigResolver', 'TriggerConfigResolver', 'SaltingConfigResolver',
+           'VibrationConfigResolver']