dynamic-systems-testing-framework 1.0.1__py3-none-any.whl

dstf/cli/baseline_cmds.py

@@ -0,0 +1,341 @@
+"""Baseline-role commands (D66): companion, soft-check, import-baseline."""
+
+import argparse
+from pathlib import Path
+
+from ..config import ConfigError
+from .common import _build_config
+
+# Positionals each action needs, in CLI order. The parser declares them all
+# ``nargs="?"`` because they're per-action — argparse can't express "required
+# for add/freeze/remove, optional for list" — so the requirement is enforced
+# here and reported as a usage error (exit 2), not a TypeError traceback out
+# of the store call.
+_COMPANION_REQUIRED = {
+    "add": ("model_id", "name", "path"),
+    "freeze": ("model_id", "name"),
+    "remove": ("model_id", "name"),
+    "list": (),
+}
+_SOFT_CHECK_REQUIRED = {
+    "remove": ("model_id", "name"),
+    "list": (),
+}
+
+
+def _require_positionals(
+    args: argparse.Namespace,
+    command: str,
+    required: dict[str, tuple[str, ...]],
+) -> None:
+    """Raise :class:`ConfigError` naming the missing positionals for
+    ``<command> <action>``. No-op when everything the action needs is set."""
+    names = required.get(args.action, ())
+    missing = [n for n in names if getattr(args, n, None) is None]
+    if not missing:
+        return
+    usage = " ".join(f"<{n}>" for n in names)
+    raise ConfigError(
+        f"{command} {args.action}: missing required argument(s): "
+        f"{', '.join(missing)}. Usage: dstf {command} {args.action} {usage}"
+    )
+
+
+class _MapError(Exception):
+    """A ``--map`` the user must fix before anything is registered."""
+
+
+def _parse_variable_map(raw: list[str] | None) -> dict[str, str]:
+    """``["x=sensor_a", ...]`` → ``{"x": "sensor_a"}``.
+
+    Two test variables MAY name one companion variable — that draws the
+    one curve on both plots. One test variable named twice is a typo: the
+    second would silently win, so it is refused instead.
+    """
+    mapping: dict[str, str] = {}
+    for item in raw or []:
+        test_var, sep, companion_var = item.partition("=")
+        if not sep or not test_var.strip() or not companion_var.strip():
+            raise _MapError(
+                f"--map {item!r} is not TEST_VAR=COMPANION_VAR — one test "
+                f"variable name, '=', one companion variable name"
+            )
+        test_var, companion_var = test_var.strip(), companion_var.strip()
+        if test_var in mapping:
+            raise _MapError(
+                f"--map names the test variable {test_var!r} twice "
+                f"({mapping[test_var]!r} and {companion_var!r}); a variable "
+                f"draws one companion curve"
+            )
+        mapping[test_var] = companion_var
+    return mapping
+
+
+def _check_variable_map(
+    store,
+    model_id: str,
+    path: Path,
+    fmt: str,
+    mapping: dict[str, str],
+) -> None:
+    """Refuse a mapping naming something neither file has.
+
+    A wrong name draws nothing and says nothing, months later, in a
+    report — so both sides are checked here, where both files are open:
+    the left against the test's stored baseline, the right against the
+    companion's own data. A path that does not resolve is left to
+    ``add_companion``, which reports it as the registration error it is,
+    and so is a model with no stored baseline at all: there is nothing to
+    check the left-hand names against, and "no primary baseline" is the
+    error the user needs, not an empty list of the variables one holds.
+    """
+    if not mapping:
+        return
+    from ..reporting.overlay_loader import companion_variable_names
+
+    reference = store.get_reference(model_id)
+    if reference is None:
+        return
+    known = {
+        entry.get("name")
+        for entry in reference.get("variables") or []
+        if isinstance(entry, dict)
+    }
+    unknown = sorted(v for v in mapping if v not in known)
+    if unknown:
+        raise _MapError(
+            f"--map names {', '.join(repr(v) for v in unknown)}, which the "
+            f"baseline of {model_id} does not hold. Its variables are: "
+            f"{', '.join(sorted(n for n in known if n))}"
+        )
+
+    if not path.exists():
+        return
+    try:
+        companion_vars = set(companion_variable_names(path, fmt))
+    except Exception as e:
+        raise _MapError(f"cannot read {path} to check --map against it: {e}") from e
+    missing = sorted(set(mapping.values()) - companion_vars)
+    if missing:
+        raise _MapError(
+            f"--map names {', '.join(repr(v) for v in missing)}, which "
+            f"{path} does not hold. Its variables are: "
+            f"{', '.join(sorted(companion_vars))}"
+        )
+
+
+def _print_companions(companions: dict) -> None:
+    """One line per companion, plus one per mapped variable under it."""
+    for name, co in companions.items():
+        extra = co.path if co.kind == "external" else co.data_file
+        print(f"  {name:<30} kind={co.kind:<8} format={co.format:<5} {extra}")
+        for test_var, companion_var in co.variables.items():
+            print(f"  {'':<30} {test_var} -> {companion_var}")
+
+
+def cmd_companion(args: argparse.Namespace) -> int:
+    """Manage plot-only companion references (D66).
+
+    Companions are experimental / analytical / rig data shown as plot
+    overlays but never scored against. Four actions:
+      add <model> <name> <path> [--format csv|json] [--map TEST=COMPANION]
+      list [<model>]
+      freeze <model> <name>
+      remove <model> <name>
+
+    ``--map`` is repeatable and records which of the companion's variables
+    draws on which of the test's — the answer the report reads instead of
+    guessing from spellings. Two test variables may name one companion
+    variable; one test variable may not be named twice. Only ``add``
+    writes the record, so only ``add`` accepts ``--map``.
+    """
+    _require_positionals(args, "companion", _COMPANION_REQUIRED)
+    if args.action != "add" and getattr(args, "variable_map", None):
+        # argparse cannot scope an option to one value of a positional
+        # choice, so the refusal lives here — silently ignoring it would
+        # leave the user believing a mapping had been recorded.
+        raise ConfigError(
+            f"companion {args.action}: --map is only accepted by "
+            f"`companion add`, which writes the mapping. To change one, "
+            f"re-add the companion with the --map you want."
+        )
+    config = _build_config(args)
+    from ..storage.reference_store import ReferenceStore
+    from ..storage.roles import infer_companion_format
+
+    store = ReferenceStore(config)
+    action = args.action
+
+    if action == "add":
+        path = Path(args.path)
+        fmt = getattr(args, "format", None) or infer_companion_format(str(path))
+        try:
+            mapping = _parse_variable_map(getattr(args, "variable_map", None))
+            _check_variable_map(store, args.model_id, path, fmt, mapping)
+            ok = store.add_companion(
+                args.model_id,
+                args.name,
+                path=path,
+                format=getattr(args, "format", None),
+                variables=mapping,
+            )
+        except (_MapError, ValueError, FileNotFoundError) as e:
+            print(f"Error: {e}")
+            return 1
+        print(
+            f"Registered companion {args.name!r} → {args.path}"
+            if ok
+            else f"Failed to register companion {args.name!r}"
+        )
+        return 0 if ok else 1
+
+    if action == "list":
+        model_id = getattr(args, "model_id", None)
+        if model_id:
+            companions = store.get_companions(model_id)
+            if not companions:
+                print(f"No companions registered for {model_id}")
+                return 0
+            print(f"Companions for {model_id}:")
+            _print_companions(companions)
+            return 0
+        # All models
+        total = 0
+        for _tid, entry in sorted(store.index.all_tests().items()):
+            if entry["status"] == "obsolete":
+                continue
+            companions = store.get_companions(entry["model_id"])
+            if companions:
+                print(f"{entry['model_id']}:")
+                _print_companions(companions)
+                total += len(companions)
+        if total == 0:
+            print("No companions registered.")
+        return 0
+
+    if action == "freeze":
+        try:
+            ok = store.freeze_companion(args.model_id, args.name)
+        except FileNotFoundError as e:
+            print(f"Error: {e}")
+            return 1
+        if ok:
+            print(f"Frozen companion {args.name!r} for {args.model_id}")
+        else:
+            print(f"Companion {args.name!r} is already frozen or does not exist")
+        return 0
+
+    if action == "remove":
+        ok = store.remove_companion(args.model_id, args.name)
+        if ok:
+            print(f"Removed companion {args.name!r} from {args.model_id}")
+        else:
+            print(f"Companion {args.name!r} not found on {args.model_id}")
+        return 0 if ok else 1
+
+    return 1
+
+
+def cmd_soft_check(args: argparse.Namespace) -> int:
+    """Manage soft_check baselines (advisory cross-checks)."""
+    _require_positionals(args, "soft-check", _SOFT_CHECK_REQUIRED)
+    config = _build_config(args)
+    from ..storage.reference_store import ReferenceStore
+
+    store = ReferenceStore(config)
+    action = args.action
+
+    if action == "list":
+        model_id = getattr(args, "model_id", None)
+        if model_id:
+            checks = store.get_soft_checks(model_id)
+            if not checks:
+                print(f"No soft_checks registered for {model_id}")
+                return 0
+            print(f"Soft_checks for {model_id}:")
+            for name, bl in checks.items():
+                origin = (
+                    bl.provenance.get("source") or bl.provenance.get("origin") or "-"
+                )
+                print(f"  {name:<30} n_vars={len(bl.variables):<4} origin={origin}")
+            return 0
+        total = 0
+        for _tid, entry in sorted(store.index.all_tests().items()):
+            if entry["status"] == "obsolete":
+                continue
+            checks = store.get_soft_checks(entry["model_id"])
+            if checks:
+                print(f"{entry['model_id']}:")
+                for name, bl in checks.items():
+                    origin = (
+                        bl.provenance.get("source")
+                        or bl.provenance.get("origin")
+                        or "-"
+                    )
+                    print(f"  {name:<30} n_vars={len(bl.variables):<4} origin={origin}")
+                total += len(checks)
+        if total == 0:
+            print("No soft_checks registered.")
+        return 0
+
+    if action == "remove":
+        ok = store.remove_soft_check(args.model_id, args.name)
+        if ok:
+            print(f"Removed soft_check {args.name!r} from {args.model_id}")
+        else:
+            print(f"Soft_check {args.name!r} not found on {args.model_id}")
+        return 0 if ok else 1
+
+    return 1
+
+
+def cmd_import_baseline(args: argparse.Namespace) -> int:
+    """Import another regression system's primary as a soft_check (D66).
+
+    Reads a reference-shape JSON from ``source_path`` (same schema a
+    ``ref_NNNN.json`` file uses at its top level) and stores it under
+    the local model's soft_checks with the given name.
+    """
+    import json as json_mod
+
+    config = _build_config(args)
+    from ..storage.reference_store import ReferenceStore, read_reference_file
+
+    store = ReferenceStore(config)
+    source = Path(args.import_source)
+    if not source.exists():
+        print(f"File not found: {source}")
+        return 1
+    try:
+        data = read_reference_file(source)
+    except (json_mod.JSONDecodeError, OSError) as e:
+        print(f"Failed to read {source}: {e}")
+        return 1
+
+    time = data.get("time") or []
+    variables = data.get("variables") or []
+    if not time or not variables:
+        print("Source file missing 'time' or 'variables' — cannot import")
+        return 1
+
+    provenance = dict(data.get("provenance") or {})
+    provenance.setdefault("origin", "import-baseline")
+    provenance.setdefault("imported_from", str(source))
+
+    try:
+        ok = store.add_soft_check(
+            args.model_id,
+            args.name,
+            time=time,
+            variables=variables,
+            diagnostics=data.get("diagnostics"),
+            simulation=data.get("simulation"),
+            statistics=data.get("statistics"),
+            provenance=provenance,
+        )
+    except (ValueError, FileNotFoundError) as e:
+        print(f"Error: {e}")
+        return 1
+    if ok:
+        print(f"Imported {source} → soft_check {args.name!r} on {args.model_id}")
+    return 0 if ok else 1

dstf/cli/common.py

@@ -0,0 +1,231 @@
+"""Shared CLI plumbing: config building, test filtering, review-category
+parsing. Everything here has callers in more than one command module."""
+
+import argparse
+import fnmatch
+import sys
+from collections.abc import Iterable
+from pathlib import Path
+
+from ..comparison.types import TestComparison
+from ..config import Config, ConfigError
+from ..discovery.test_registry import TestModel
+from ..reporting.status import (
+    FAIL,
+    NO_REF,
+    NOT_RUN,
+    PASS,
+    SIM_FAIL,
+    classify,
+    exit_code,
+    nothing_ran,
+)
+
+
+def _build_config(args: argparse.Namespace) -> Config:
+    """Build Config from parsed CLI arguments."""
+    kwargs = {}
+    if args.source_path:
+        kwargs["source_path"] = Path(args.source_path)
+    if hasattr(args, "config") and args.config:
+        kwargs["config_file"] = Path(args.config)
+    if hasattr(args, "reference_root") and args.reference_root:
+        kwargs["reference_root"] = Path(args.reference_root)
+    if hasattr(args, "backend") and args.backend:
+        kwargs["backend_name"] = args.backend
+    if hasattr(args, "backend_path") and args.backend_path:
+        kwargs["backend_path"] = args.backend_path
+    if hasattr(args, "show_ide") and args.show_ide:
+        kwargs["show_ide"] = True
+    if hasattr(args, "work_dir") and args.work_dir:
+        kwargs["work_dir"] = Path(args.work_dir)
+    if hasattr(args, "parallel") and args.parallel:
+        kwargs["parallel"] = args.parallel
+    if hasattr(args, "batch_size") and args.batch_size is not None:
+        kwargs["batch_size"] = args.batch_size
+    if hasattr(args, "dymola_interface") and args.dymola_interface:
+        kwargs["dymola_interface_path"] = Path(args.dymola_interface)
+    # `is not None`, NOT truthiness — `--tolerance 0` is the strictest
+    # possible setting and Config/comparator honor an explicit 0.0
+    # (comparator.py resolve chain, config.py file-tolerance precedence).
+    if hasattr(args, "tolerance") and args.tolerance is not None:
+        kwargs["tolerance"] = args.tolerance
+    if hasattr(args, "default_points") and args.default_points:
+        kwargs["default_points"] = args.default_points
+    if hasattr(args, "timeout") and args.timeout:
+        kwargs["timeout"] = args.timeout
+    if hasattr(args, "test_spec") and args.test_spec:
+        kwargs["test_spec_file"] = Path(args.test_spec)
+    return Config(**kwargs)
+
+
+def _split_filter_list(spec: str) -> list[str]:
+    """Split a comma-separated filter list on commas NOT inside ``[]``.
+
+    review 2026-07-06 finding 64: naive comma-splitting broke fnmatch
+    character classes like ``Test[A,B]*``. Small scanner: a comma inside an
+    open ``[`` class belongs to the class. No escape mechanism — an
+    unclosed ``[`` makes the rest of the string one pattern (fnmatch treats
+    an unmatched ``[`` as a literal anyway).
+    """
+    parts: list[str] = []
+    buf: list[str] = []
+    in_class = False
+    for ch in spec:
+        if ch == "," and not in_class:
+            parts.append("".join(buf))
+            buf = []
+            continue
+        if ch == "[" and not in_class:
+            in_class = True
+        elif ch == "]" and in_class:
+            in_class = False
+        buf.append(ch)
+    parts.append("".join(buf))
+    return [p.strip() for p in parts if p.strip()]
+
+
+def _resolve_filter_patterns(spec: str) -> list[str]:
+    """Resolve a --filter argument into a list of glob patterns.
+
+    Accepts:
+      - "Foo.Bar.*"           — single glob
+      - "Foo.A,Foo.B,Foo.C"   — comma-separated globs (commas inside []
+                                character classes are part of the class)
+      - "@path/to/file.txt"   — file with one pattern per line; '#' starts a comment
+    """
+    if spec.startswith("@"):
+        path = Path(spec[1:])
+        if not path.exists():
+            # ConfigError, not FileNotFoundError: a mistyped @file is a user
+            # error and main() turns it into one clean "Config error: ..."
+            # line and exit 2, never a traceback.
+            raise ConfigError(f"Filter file not found: {path}")
+        try:
+            text = path.read_text(encoding="utf-8")
+        except OSError as e:
+            raise ConfigError(f"Cannot read filter file {path}: {e}") from e
+        patterns = []
+        for raw in text.splitlines():
+            line = raw.split("#", 1)[0].strip()
+            if line:
+                patterns.append(line)
+        return patterns
+    return _split_filter_list(spec)
+
+
+def _filter_tests(
+    tests: list[TestModel],
+    pattern: str | None = None,
+    package: str | None = None,
+) -> list[TestModel]:
+    """Filter tests by glob/list/file on model_id, plus optional package prefix."""
+    filtered = tests
+    if package:
+        # review 2026-07-06 finding 65: match the package itself or children
+        # across a dot boundary — "MyLib.Fluid" must not also select
+        # "MyLib.FluidExperimental.*" (raw startswith did).
+        filtered = [
+            t
+            for t in filtered
+            if t.model_id == package or t.model_id.startswith(package + ".")
+        ]
+    if pattern:
+        patterns = _resolve_filter_patterns(pattern)
+        # review 2026-07-06 finding 63: fnmatchcase — fnmatch.fnmatch is
+        # case-insensitive on Windows, so the same filter selected different
+        # tests across the two OS partitions this project maintains.
+        filtered = [
+            t
+            for t in filtered
+            if any(fnmatch.fnmatchcase(t.model_id, p) for p in patterns)
+        ]
+    return filtered
+
+
+def _find_orphan_manifest_entries(config, all_tests: list) -> dict[str, str]:
+    """Return {test_key: model_id} for manifest entries whose model_id is no
+    longer in the discovered tests. These usually indicate renamed/removed
+    models — they don't break correctness (compare_all skips them) but their
+    work dirs and report dirs accumulate on disk.
+    """
+    manifest_path = config.work_dir / "batch_manifest.json"
+    if not manifest_path.exists():
+        return {}
+    from ..backends import BatchManifest
+
+    manifest = BatchManifest.load(manifest_path)
+    discovered = {t.model_id for t in all_tests}
+    return {
+        tk: entry["model_id"]
+        for tk, entry in manifest.manifest.items()
+        if entry["model_id"] not in discovered
+    }
+
+
+_VALID_REVIEW_FILTERS = {
+    "all",
+    "failed",
+    "no-baseline",
+    "warnings",
+    "sim-failed",
+    "passed",
+}
+
+
+def _exit_code(comparisons: Iterable[TestComparison]) -> int:
+    """`run` and `compare`'s exit code on every route (``exit_code``), and
+    when it is 1 because nothing ran, ONE stderr line saying so: a junit or
+    html report, or ``--report``'s pages, exited 1 without a word (D145
+    §5). With no test in scope at all there is nothing to count, and the
+    line says that instead of "(0 NOT RUN)"."""
+    comps = list(comparisons)
+    if not comps:
+        print("Nothing ran: no test in scope.", file=sys.stderr)
+    elif nothing_ran(comps):
+        print(
+            f"Nothing ran: no test in scope has a result ({len(comps)} "
+            f"{NOT_RUN}) — exit 1.",
+            file=sys.stderr,
+        )
+    return exit_code(comps)
+
+
+def _parse_review_filter(filter_str: str) -> set[str]:
+    """Parse and validate the review filter string."""
+    filters = {f.strip() for f in filter_str.split(",") if f.strip()}
+    invalid = filters - _VALID_REVIEW_FILTERS
+    if invalid:
+        raise argparse.ArgumentTypeError(
+            f"Invalid review filter(s): {', '.join(sorted(invalid))}. "
+            f"Valid: {', '.join(sorted(_VALID_REVIEW_FILTERS - {'all'}))}"
+        )
+    if "all" in filters:
+        return {"all"}
+    return filters
+
+
+def _should_review(comp, filters: set[str]) -> bool:
+    """Check if a comparison matches the active review filters (`-i` and
+    `--rerun`). A category selects by the ONE classifier's status
+    (``reporting.status.classify``), the status every surface shows."""
+    if "all" in filters:
+        return True
+    status = classify(comp)
+    # NOT RUN too: nothing succeeded (D145).
+    if "sim-failed" in filters and status in (SIM_FAIL, NOT_RUN):
+        return True
+    # A test the comparison FAILED is "failed" and nothing else, baseline or
+    # not, so "no-baseline" does not select it: one the comparison refused,
+    # or one with no reference that fails a baseline-free criterion (range /
+    # declared events / declared peaks — review 2026-07-06, finding 5).
+    # "failed" selects both. The review loop then offers the second
+    # [a]ccept — accepting it writes a reference — and the first none.
+    if "no-baseline" in filters and not comp.has_reference and status != FAIL:
+        return True
+    if "failed" in filters and status == FAIL:
+        return True
+    if "warnings" in filters and comp.warnings:
+        return True
+    # NO_REF too: nothing was scored, and `compare_one` marks it passed.
+    return bool("passed" in filters and status in (PASS, NO_REF) and not comp.warnings)