pytesprocess 0.1.1__py3-none-any.whl

pytesprocess/config/resolvers/salting.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import copy
+import warnings
+
+from ..selectors import expand_channel_selector
+from ..warnings import ConfigDeprecationWarning
+from .common import deep_merge
+
+
+class SaltingConfigResolver:
+    """Resolve salting configuration into the legacy runtime shape.
+
+    Salt pulse construction depends on a template and, optionally, dPdI.  PSD
+    and CSD tags were historically threaded through the API but never used by
+    the salting calculation; they are now accepted only as deprecated no-ops.
+    """
+
+    def __init__(self, config, resources, available_channels, version=1):
+        self._config = copy.deepcopy(config or {})
+        self._resources = copy.deepcopy(resources or {})
+        self._available_channels = list(available_channels or [])
+        self._version = int(version)
+
+    def resolve(self):
+        overall = copy.deepcopy(self._config.get('global', {}))
+        for key, value in self._resources.items():
+            overall.setdefault(key, value)
+        channels = self._config.get('channels', {})
+        if not isinstance(channels, dict):
+            raise ValueError('ERROR: salting.channels must be a mapping.')
+
+        assignments = {}
+        physical_list = []
+        for selector, block in channels.items():
+            if not isinstance(block, dict):
+                raise ValueError(
+                    f'ERROR: Salting configuration for "{selector}" must be a mapping.'
+                )
+            for match in expand_channel_selector(selector, self._available_channels):
+                assignments.setdefault(match.target, []).append(
+                    (match.specificity, selector, copy.deepcopy(block),
+                     match.physical_channels)
+                )
+
+        resolved = {}
+        for target, items in assignments.items():
+            items.sort(key=lambda item: item[0])
+            merged = {}
+            seen = {}
+            physical = ()
+            for specificity, selector, block, physical_channels in items:
+                if specificity in seen:
+                    raise ValueError(
+                        f'ERROR: Salting selectors "{seen[specificity]}" and "{selector}" '
+                        f'both match "{target}" with equal precedence.'
+                    )
+                seen[specificity] = selector
+                merged = deep_merge(merged, block)
+                physical = physical_channels
+            if merged.get('disable', False) or merged.get('run') is False:
+                continue
+
+            # ``noise_tag`` is normalized to ``csd_tag`` by the compatibility
+            # loader. Neither was ever consumed by salting. Remove it from the
+            # resolved runtime configuration so new code cannot accidentally
+            # imply that CSD participates in pulse generation.
+            if 'csd_tag' in merged:
+                warnings.warn(
+                    f'Salting configuration "{target}.csd_tag" is deprecated '
+                    'and ignored. Salting uses template_tag and optional dPdI.',
+                    ConfigDeprecationWarning,
+                    stacklevel=3,
+                )
+                merged.pop('csd_tag', None)
+
+            merged.pop('disable', None)
+            merged['run'] = True
+            resolved[target] = merged
+            physical_list.extend(physical)
+
+        return {
+            'overall': overall,
+            'channels': resolved,
+            'channel_list': list(dict.fromkeys(physical_list)),
+        }

pytesprocess/config/resolvers/trigger.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import copy
+from ..selectors import expand_channel_selector
+from .common import deep_merge
+
+
+class TriggerConfigResolver:
+    def __init__(self, config, resources, available_channels, version=1):
+        self._config = copy.deepcopy(config or {})
+        self._resources = copy.deepcopy(resources or {})
+        self._available_channels = list(available_channels or [])
+        self._version = int(version)
+
+    def resolve(self):
+        overall = copy.deepcopy(self._config.get('global', {}))
+        for key, value in self._resources.items():
+            overall.setdefault(key, value)
+        channels = self._config.get('channels', {})
+        if not isinstance(channels, dict):
+            raise ValueError('ERROR: trigger.channels must be a mapping.')
+
+        assignments = {}
+        physical_list = []
+        for selector, block in channels.items():
+            if not isinstance(block, dict):
+                raise ValueError(f'ERROR: Trigger configuration for "{selector}" must be a mapping.')
+            for match in expand_channel_selector(selector, self._available_channels):
+                assignments.setdefault(match.target, []).append((match.specificity, selector,
+                                                                  copy.deepcopy(block),
+                                                                  match.physical_channels))
+
+        merged_channels = {}
+        for target, items in assignments.items():
+            items.sort(key=lambda item: item[0])
+            merged = {}
+            seen = {}
+            physical = ()
+            for specificity, selector, block, physical_channels in items:
+                if specificity in seen:
+                    raise ValueError(
+                        f'ERROR: Trigger selectors "{seen[specificity]}" and "{selector}" '
+                        f'both match "{target}" with equal precedence.'
+                    )
+                seen[specificity] = selector
+                merged = deep_merge(merged, block)
+                physical = physical_channels
+            if merged.get('disable', False) or merged.get('run') is False:
+                continue
+            physical_list.extend(physical)
+            merged_channels[target] = merged
+
+        trigger_channel_dict = {}
+        for channel, channel_cfg in merged_channels.items():
+            cfg = copy.deepcopy(channel_cfg)
+            trigger_channel = cfg.pop('trigger_channel', channel)
+
+            # One direct trigger config, or named trigger algorithms under channel.
+            if 'run' in cfg or not any(isinstance(v, dict) for v in cfg.values()):
+                run = cfg.pop('run', True)
+                if run:
+                    # Runtime trigger code still expects an explicit ``run``
+                    # flag.  Public YAML semantics are simpler: omitted means
+                    # enabled, while only ``run: false`` disables the block.
+                    cfg['run'] = True
+                    cfg['channel_name'] = channel
+                    trigger_channel_dict[trigger_channel] = cfg
+                continue
+
+            for algo, algo_cfg in cfg.items():
+                if not isinstance(algo_cfg, dict):
+                    continue
+                algo_cfg = copy.deepcopy(algo_cfg)
+                if not algo_cfg.pop('run', True):
+                    continue
+                algo_cfg['run'] = True
+                algo_cfg['channel_name'] = channel
+                trigger_channel_dict[f'{algo}_{trigger_channel}'] = algo_cfg
+
+        return {
+            'overall': overall,
+            'channels': trigger_channel_dict,
+            'channel_list': list(dict.fromkeys(physical_list)),
+        }

pytesprocess/config/selectors.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from fnmatch import fnmatchcase
+
+
+_COMPOSITE_SEPARATORS = ('|', '+', '-')
+_GLOB_CHARS = ('*', '?', '[')
+
+
+@dataclass(frozen=True)
+class SelectorMatch:
+    target: str
+    physical_channels: tuple[str, ...]
+    specificity: int
+    selector: str
+
+
+def _has_glob(text: str) -> bool:
+    return any(char in text for char in _GLOB_CHARS)
+
+
+def _split_composite(expr: str) -> tuple[list[str], str | None]:
+    separators = [sep for sep in _COMPOSITE_SEPARATORS if sep in expr]
+    if not separators:
+        return [expr], None
+    if len(separators) > 1:
+        raise ValueError(
+            f'ERROR: Channel expression "{expr}" mixes separators {separators}. '
+            'Use one of "|", "+", or "-" per multi-channel expression.'
+        )
+    sep = separators[0]
+    return expr.split(sep), sep
+
+
+def expand_channel_selector(selector: str, available_channels) -> list[SelectorMatch]:
+    """Expand one config selector while preserving composite-channel semantics.
+
+    Rules:
+      * ``all`` selects all physical channels.
+      * glob patterns select physical channels only.
+      * comma separates independent physical selectors.
+      * ``|``, ``+`` and ``-`` denote one exact composite expression and may
+        not contain globs or commas.
+    """
+    if not isinstance(selector, str) or not selector.strip():
+        raise ValueError('ERROR: Channel selector must be a non-empty string.')
+
+    available = [str(chan) for chan in available_channels]
+    selector = selector.replace(' ', '')
+
+    # Multi-channel expressions are exact and never wildcard-expanded.
+    parts, sep = _split_composite(selector)
+    if sep is not None:
+        if ',' in selector:
+            raise ValueError(
+                f'ERROR: Composite channel expression "{selector}" cannot contain comma.'
+            )
+        if any(_has_glob(part) for part in parts):
+            raise ValueError(
+                f'ERROR: Wildcards are not allowed inside composite channel expression '
+                f'"{selector}". List the physical channels explicitly.'
+            )
+        missing = [part for part in parts if part not in available]
+        if missing:
+            raise ValueError(
+                f'ERROR: Unknown channel(s) {missing} in expression "{selector}". '
+                f'Available channels = {available}'
+            )
+        return [SelectorMatch(selector, tuple(parts), 40, selector)]
+
+    tokens = selector.split(',')
+    is_comma_group = len(tokens) > 1
+    results: list[SelectorMatch] = []
+    seen = set()
+
+    for token in tokens:
+        if not token:
+            raise ValueError(f'ERROR: Invalid empty channel in selector "{selector}".')
+
+        if token == 'all':
+            matched = available
+            specificity = 0 if not is_comma_group else 20
+        elif _has_glob(token):
+            matched = [chan for chan in available if fnmatchcase(chan, token)]
+            if not matched:
+                raise ValueError(
+                    f'ERROR: Channel wildcard "{token}" did not match any raw channel. '
+                    f'Available channels = {available}'
+                )
+            specificity = 10 if not is_comma_group else 20
+        else:
+            if token not in available:
+                raise ValueError(
+                    f'ERROR: Channel "{token}" not found in raw data. '
+                    f'Available channels = {available}'
+                )
+            matched = [token]
+            specificity = 30 if not is_comma_group else 20
+
+        for chan in matched:
+            key = (chan, specificity)
+            if key in seen:
+                continue
+            seen.add(key)
+            results.append(SelectorMatch(chan, (chan,), specificity, selector))
+
+    return results

pytesprocess/config/validation.py

@@ -0,0 +1,314 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+
+_FEATURE_OF_PREFIXES = ('of1x1', 'of1x2x2', 'of1x3x3', 'ofnxm', 'ofnxmx2')
+
+_SUPPORTED_WORKFLOWS = {
+    'trigger', 'feature', 'salting', 'filter', 'ivsweep',
+    'didv', 'noise', 'template',
+}
+
+
+def normalize_workflows(workflows):
+    if workflows is None:
+        return None
+    if isinstance(workflows, str):
+        workflows = [workflows]
+    names = []
+    for workflow in workflows:
+        name = str(workflow).strip().lower()
+        if name not in _SUPPORTED_WORKFLOWS:
+            raise ValueError(
+                f'ERROR: Unknown processing workflow "{workflow}". '
+                f'Available workflows = {sorted(_SUPPORTED_WORKFLOWS)}'
+            )
+        if name not in names:
+            names.append(name)
+    return names
+
+
+def validate_resolved_workflow(name, config, check_resources=False):
+    """Validate one already-resolved workflow configuration.
+
+    Validation here checks the contract between the configuration layer and
+    processing classes.  It intentionally does not read raw records, open
+    FilterData contents, or run algorithms; those remain runtime checks.
+    """
+    if not isinstance(config, dict):
+        raise ValueError(
+            f'ERROR: Resolved {name} configuration must be a mapping, '
+            f'got {type(config).__name__}.'
+        )
+
+    if name == 'trigger':
+        return _validate_trigger(config, check_resources=check_resources)
+    if name == 'feature':
+        return _validate_feature(config, check_resources=check_resources)
+    if name == 'salting':
+        return _validate_salting(config, check_resources=check_resources)
+
+    # FilterDataProcessing and IVSweepProcessing currently own their more
+    # specialized validation.  The configuration manager still verifies that
+    # the section has the expected mapping type so a future CLI can validate
+    # all requested workflow sections through one entry point.
+    return {
+        'configured_items': len(config),
+    }
+
+
+def _require_mapping(config, key, workflow):
+    value = config.get(key, {})
+    if not isinstance(value, dict):
+        raise ValueError(
+            f'ERROR: Resolved {workflow}.{key} configuration must be a mapping.'
+        )
+    return value
+
+
+def _require_filter_file(overall, workflow, check_resources=False):
+    filter_file = overall.get('filter_file')
+    if filter_file is None or not str(filter_file).strip():
+        raise ValueError(
+            f'ERROR: Filter file missing for {workflow} processing. '
+            'Define resources.filter_file (or legacy filter_file).'
+        )
+    if check_resources:
+        path = Path(str(filter_file)).expanduser()
+        if not path.is_file():
+            raise ValueError(
+                f'ERROR: Filter file "{filter_file}" required by {workflow} '
+                'processing does not exist.'
+            )
+    return str(filter_file)
+
+
+def _validate_trigger(config, check_resources=False):
+    overall = _require_mapping(config, 'overall', 'trigger')
+    channels = _require_mapping(config, 'channels', 'trigger')
+    channel_list = config.get('channel_list', [])
+    if not isinstance(channel_list, list):
+        raise ValueError('ERROR: Resolved trigger.channel_list must be a list.')
+    if not channels:
+        raise ValueError(
+            'ERROR: Trigger processing was requested but no trigger channels '
+            'are enabled in the configuration.'
+        )
+    if not channel_list:
+        raise ValueError(
+            'ERROR: Trigger processing was requested but no physical raw '
+            'channels were selected.'
+        )
+
+    for name, block in channels.items():
+        if not isinstance(block, dict):
+            raise ValueError(
+                f'ERROR: Resolved trigger configuration "{name}" must be a mapping.'
+            )
+        if block.get('run') is not True:
+            raise ValueError(
+                f'ERROR: Resolved trigger configuration "{name}" does not have '
+                'run: true. Disabled trigger blocks should have been removed by '
+                'the resolver.'
+            )
+        if not block.get('channel_name'):
+            raise ValueError(
+                f'ERROR: Resolved trigger configuration "{name}" is missing '
+                'channel_name.'
+            )
+
+    filter_file = _require_filter_file(
+        overall, 'trigger', check_resources=check_resources)
+    return {
+        'configured_channels': len(channels),
+        'physical_channels': len(channel_list),
+        'filter_file': filter_file,
+    }
+
+
+def _validate_feature(config, check_resources=False):
+    overall = _require_mapping(config, 'overall', 'feature')
+    channels = _require_mapping(config, 'channels', 'feature')
+    channel_list = config.get('channel_list', [])
+    if not isinstance(channel_list, list):
+        raise ValueError('ERROR: Resolved feature.channel_list must be a list.')
+    if not channels:
+        raise ValueError(
+            'ERROR: Feature processing was requested but no feature channels '
+            'or channel expressions are enabled in the configuration.'
+        )
+    if not channel_list:
+        raise ValueError(
+            'ERROR: Feature processing was requested but no physical raw '
+            'channels were selected.'
+        )
+
+    algorithm_count = 0
+    trace_configs = set()
+    requires_filter_file = False
+    for channel, block in channels.items():
+        if not isinstance(block, dict):
+            raise ValueError(
+                f'ERROR: Resolved feature configuration for "{channel}" '
+                'must be a mapping.'
+            )
+        for algorithm, params in block.items():
+            if not isinstance(params, dict):
+                # feature_channel, weight_*, and other channel-level metadata
+                # intentionally remain non-dict entries.
+                continue
+            algorithm_count += 1
+            base_algorithm = params.get('base_algorithm', algorithm)
+            if str(base_algorithm).startswith(_FEATURE_OF_PREFIXES):
+                requires_filter_file = True
+            if params.get('run') is not True:
+                raise ValueError(
+                    f'ERROR: Resolved feature algorithm "{channel}/{algorithm}" '
+                    'does not have run: true. Disabled algorithms should have '
+                    'been removed by the resolver.'
+                )
+            if 'nb_samples' not in params or 'nb_pretrigger_samples' not in params:
+                raise ValueError(
+                    f'ERROR: Resolved feature algorithm "{channel}/{algorithm}" '
+                    'is missing trace-length fields.'
+                )
+            nb_samples = params['nb_samples']
+            nb_pre = params['nb_pretrigger_samples']
+            if (nb_samples is None) != (nb_pre is None):
+                raise ValueError(
+                    f'ERROR: Trace length and pretrigger length must either both '
+                    f'be set or both be omitted for "{channel}/{algorithm}".'
+                )
+            if nb_samples is not None:
+                if nb_samples <= 0 or nb_pre < 0 or nb_pre > nb_samples:
+                    raise ValueError(
+                        f'ERROR: Invalid resolved trace/pretrigger lengths for '
+                        f'"{channel}/{algorithm}": {nb_samples}/{nb_pre} samples.'
+                    )
+                trace_configs.add((int(nb_samples), int(nb_pre)))
+
+    if algorithm_count == 0:
+        raise ValueError(
+            'ERROR: Feature processing was requested but no feature algorithms '
+            'are enabled in the configuration.'
+        )
+
+    filter_file = overall.get('filter_file')
+    if requires_filter_file:
+        filter_file = _require_filter_file(
+            overall, 'feature', check_resources=check_resources)
+    elif check_resources and filter_file:
+        path = Path(str(filter_file)).expanduser()
+        if not path.is_file():
+            raise ValueError(
+                f'ERROR: Optional feature filter file "{filter_file}" does not exist.'
+            )
+
+    return {
+        'configured_channels': len(channels),
+        'physical_channels': len(channel_list),
+        'algorithms': algorithm_count,
+        'trace_configurations': len(trace_configs),
+        'filter_file': filter_file,
+        'requires_filter_file': requires_filter_file,
+    }
+
+
+def _validate_salting(config, check_resources=False):
+    overall = _require_mapping(config, 'overall', 'salting')
+    channels = _require_mapping(config, 'channels', 'salting')
+    channel_list = config.get('channel_list', [])
+    if not isinstance(channel_list, list):
+        raise ValueError('ERROR: Resolved salting.channel_list must be a list.')
+    if not channels:
+        raise ValueError(
+            'ERROR: Salting was requested but no salting channels or channel '
+            'expressions are enabled in the configuration.'
+        )
+    if not channel_list:
+        raise ValueError(
+            'ERROR: Salting was requested but no physical raw channels were selected.'
+        )
+    for channel, block in channels.items():
+        if not isinstance(block, dict):
+            raise ValueError(
+                f'ERROR: Resolved salting configuration for "{channel}" '
+                'must be a mapping.'
+            )
+        if block.get('run') is not True:
+            raise ValueError(
+                f'ERROR: Resolved salting configuration "{channel}" does not '
+                'have run: true. Disabled blocks should have been removed by '
+                'the resolver.'
+            )
+        if not block.get('template_tag'):
+            raise ValueError(
+                f'ERROR: Salting channel "{channel}" requires template_tag.'
+            )
+        dpdi_tag = block.get('dpdi_tag')
+        dpdi_poles = block.get('dpdi_poles')
+        if (dpdi_tag is None) != (dpdi_poles is None):
+            raise ValueError(
+                f'ERROR: Salting channel "{channel}" requires dpdi_tag and '
+                'dpdi_poles to be both set or both omitted.'
+            )
+        if 'csd_tag' in block or 'noise_tag' in block:
+            raise ValueError(
+                f'ERROR: Resolved salting channel "{channel}" still contains '
+                'a noise/CSD tag. These are not used by salting.'
+            )
+
+    if overall.get('energies') is not None and overall.get('dm_pdf_file') is not None:
+        raise ValueError(
+            'ERROR: Salting configuration must use fixed energies or dm_pdf_file, not both.'
+        )
+    if overall.get('nsalt') is not None and int(overall['nsalt']) <= 0:
+        raise ValueError('ERROR: salting.global.nsalt must be positive.')
+    if overall.get('random_seed') is not None:
+        try:
+            int(overall['random_seed'])
+        except (TypeError, ValueError):
+            raise ValueError('ERROR: salting.global.random_seed must be an integer or null.')
+
+    filter_file = _require_filter_file(
+        overall, 'salting', check_resources=check_resources)
+    if check_resources and overall.get('didv_file'):
+        didv_file = Path(str(overall['didv_file'])).expanduser()
+        if not didv_file.is_file():
+            raise ValueError(
+                f'ERROR: dIdV file "{overall["didv_file"]}" required by salting '
+                'does not exist.'
+            )
+    if check_resources and overall.get('dm_pdf_file'):
+        pdf_file = Path(str(overall['dm_pdf_file'])).expanduser()
+        if not pdf_file.is_file():
+            raise ValueError(
+                f'ERROR: DM PDF file "{overall["dm_pdf_file"]}" required by '
+                'salting does not exist.'
+            )
+    return {
+        'configured_channels': len(channels),
+        'physical_channels': len(channel_list),
+        'filter_file': filter_file,
+    }
+
+
+def format_validation_summary(summary):
+    lines = ['Configuration validation:']
+    for workflow, info in summary.get('workflows', {}).items():
+        details = []
+        if 'configured_channels' in info:
+            details.append(f'{info["configured_channels"]} configured channel(s)')
+        if 'physical_channels' in info:
+            details.append(f'{info["physical_channels"]} physical channel(s)')
+        if 'algorithms' in info:
+            details.append(f'{info["algorithms"]} algorithm(s)')
+        if 'trace_configurations' in info:
+            details.append(f'{info["trace_configurations"]} trace configuration(s)')
+        suffix = f' ({", ".join(details)})' if details else ''
+        lines.append(f'  {workflow:<10} OK{suffix}')
+    if not summary.get('workflows'):
+        lines.append('  no workflows requested')
+    return '\n'.join(lines)

pytesprocess/config/warnings.py

@@ -0,0 +1,2 @@
+class ConfigDeprecationWarning(FutureWarning):
+    """Visible warning for deprecated processing-configuration syntax."""

pytesprocess/core/__init__.py

@@ -0,0 +1,10 @@
+from .algorithms import *
+from .oftrigger import *
+from .eventbuilder import *
+from .filterdata import *
+from .noise import *
+from .template import *
+from .ivsweep import *
+from .didv import *
+from .noisemodel import *
+from .filterbuilder import *