pyflightstream 0.2.0__py3-none-any.whl

pyflightstream/qa/cli.py

@@ -0,0 +1,364 @@
+"""The ``pyfs-qa`` command line, first console entry point of the package.
+
+Pipeline role: drives the qa evidence workflow from a terminal on the
+licensed machine. ``pyfs-qa probe`` runs the Tier 2 command-validity
+probes for one FlightStream version and writes the compat report;
+``pyfs-qa apply-compat`` promotes database statuses from a committed
+report. ``pyfs-qa physics`` runs the Tier 3 physics regression matrix
+and writes the physics report; ``pyfs-qa update-reference`` is the only
+write path into the stored physics references and demands a reason
+string (SAD Section 11); ``pyfs-qa drift`` runs the same case set on
+two versions and diffs the aggregated coefficients (FR-27).
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from pyflightstream.commands import CommandRegistry
+from pyflightstream.qa.compat import apply_compat, write_compat_report
+from pyflightstream.qa.drift import run_drift, write_drift_report
+from pyflightstream.qa.physics import (
+    PhysicsEnvironmentError,
+    run_physics,
+    update_reference,
+    write_physics_report,
+)
+from pyflightstream.qa.probes import ProbeEnvironmentError, probe_version
+from pyflightstream.versions import resolve
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="pyfs-qa",
+        description=(
+            "Tier 2 and 3 evidence tooling for the pyflightstream command database; "
+            "probes run FlightStream locally and need a licensed machine."
+        ),
+    )
+    subparsers = parser.add_subparsers(dest="subcommand", required=True)
+
+    probe = subparsers.add_parser(
+        "probe",
+        help="run command-validity probes and write the compat report",
+    )
+    probe.add_argument(
+        "--version",
+        required=True,
+        help="target FlightStream version, canonical or alias (for example 26.120)",
+    )
+    probe.add_argument(
+        "--fs-exe",
+        required=True,
+        help="explicit path of the FlightStream executable (never guessed)",
+    )
+    probe.add_argument(
+        "--commands",
+        help="comma-separated subset to probe; default: every command with a probe spec",
+    )
+    probe.add_argument(
+        "--fsm",
+        help=(
+            "local simulation (.fsm) file for prelude tiers above none; specs needing "
+            "it are recorded unprobed when absent"
+        ),
+    )
+    probe.add_argument(
+        "--workroot",
+        default="probe_runs",
+        help="scratch root for probe scripts and logs (default probe_runs/, not committed)",
+    )
+    probe.add_argument(
+        "--timeout", type=float, default=120.0, help="per-probe wall-clock limit, seconds"
+    )
+    probe.add_argument(
+        "--report-dir",
+        default="reports/compat",
+        help="directory receiving the compat report pair (default reports/compat/)",
+    )
+    probe.add_argument(
+        "--label",
+        help="report stem suffix distinguishing several reports on one day",
+    )
+
+    apply_parser = subparsers.add_parser(
+        "apply-compat",
+        help="promote database statuses from a committed compat report",
+    )
+    apply_parser.add_argument("report", help="the compat report YAML file")
+    apply_parser.add_argument(
+        "--root",
+        default=".",
+        help="repository root; the report is cited relative to it (default .)",
+    )
+
+    physics = subparsers.add_parser(
+        "physics",
+        help="run the Tier 3 physics regression matrix and write the physics report",
+    )
+    physics.add_argument(
+        "--version",
+        required=True,
+        help="target FlightStream version, canonical or alias (for example 26.120)",
+    )
+    physics.add_argument(
+        "--fs-exe",
+        required=True,
+        help="explicit path of the FlightStream executable (never guessed)",
+    )
+    physics.add_argument(
+        "--cases",
+        help="comma-separated case subset (for example PHY-01); default: every case",
+    )
+    physics.add_argument(
+        "--workroot",
+        default="physics_runs",
+        help="scratch root for geometry, scripts, and outputs (default physics_runs/)",
+    )
+    physics.add_argument(
+        "--timeout", type=float, default=900.0, help="per-point wall-clock limit, seconds"
+    )
+    physics.add_argument(
+        "--report-dir",
+        default="reports/physics",
+        help="directory receiving the report pair (default reports/physics/)",
+    )
+    physics.add_argument(
+        "--label",
+        help="report stem suffix distinguishing several reports on one day",
+    )
+    physics.add_argument(
+        "--smi-root",
+        help="local SMI geometry root (normally _private/geometry/smi); enables the "
+        "SMI drift class, geometry never enters Git",
+    )
+
+    drift = subparsers.add_parser(
+        "drift",
+        help="run the case set on two versions and diff the coefficients (FR-27)",
+    )
+    drift.add_argument(
+        "--versions",
+        required=True,
+        help="two comma-separated versions, baseline first (for example 26.100,26.120); "
+        "the same version twice is the degenerate self-comparison",
+    )
+    drift.add_argument(
+        "--fs-exe",
+        action="append",
+        required=True,
+        metavar="VERSION=PATH",
+        help="explicit executable per version (repeatable, never guessed), "
+        "for example --fs-exe 26.100=C:/fs26100/FS.exe",
+    )
+    drift.add_argument(
+        "--cases",
+        help="comma-separated case subset (for example PHY-01); default: every case",
+    )
+    drift.add_argument(
+        "--workroot",
+        default="physics_runs",
+        help="scratch root; each version nests under its canonical name",
+    )
+    drift.add_argument(
+        "--timeout", type=float, default=900.0, help="per-point wall-clock limit, seconds"
+    )
+    drift.add_argument(
+        "--report-dir",
+        default="reports/physics",
+        help="directory receiving the report pair (default reports/physics/)",
+    )
+    drift.add_argument(
+        "--label",
+        help="report stem suffix distinguishing several reports on one day",
+    )
+    drift.add_argument(
+        "--smi-root",
+        help="local SMI geometry root (normally _private/geometry/smi); enables the "
+        "SMI drift class on both versions, geometry never enters Git",
+    )
+
+    update = subparsers.add_parser(
+        "update-reference",
+        help="update or seed one physics reference from a committed physics report",
+    )
+    update.add_argument("case", help="case identifier, for example PHY-01")
+    update.add_argument(
+        "--from-report",
+        required=True,
+        help="committed physics report YAML carrying the measured values",
+    )
+    update.add_argument(
+        "--reason",
+        required=True,
+        help="why the reference moves; recorded in the reference file (required)",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Run ``pyfs-qa``; returns the process exit code."""
+    args = _build_parser().parse_args(argv)
+    if args.subcommand == "probe":
+        return _cmd_probe(args)
+    if args.subcommand == "physics":
+        return _cmd_physics(args)
+    if args.subcommand == "drift":
+        return _cmd_drift(args)
+    if args.subcommand == "update-reference":
+        return _cmd_update_reference(args)
+    return _cmd_apply_compat(args)
+
+
+def _cmd_probe(args: argparse.Namespace) -> int:
+    canonical = resolve(args.version).canonical
+    commands = None
+    if args.commands:
+        commands = [name.strip() for name in args.commands.split(",") if name.strip()]
+    try:
+        run = probe_version(
+            canonical,
+            workroot=Path(args.workroot) / canonical,
+            fs_exe=args.fs_exe,
+            commands=commands,
+            fsm=args.fsm,
+            timeout_s=args.timeout,
+        )
+    except ProbeEnvironmentError as error:
+        print(f"probe run aborted: {error}", file=sys.stderr)
+        return 2
+    yaml_path, md_path = write_compat_report(run, args.report_dir, label=args.label)
+    counts = run.outcome_counts()
+    print(
+        f"FlightStream {run.version} ({run.fs_exe_name}): "
+        f"{counts['verified']} verified, {counts['broken']} broken, "
+        f"{counts['unprobed']} unprobed"
+    )
+    for line in run.solver_identity:
+        print(f"solver: {line}")
+    print(f"report: {yaml_path}")
+    print(f"report: {md_path}")
+    return 0
+
+
+def _cmd_physics(args: argparse.Namespace) -> int:
+    canonical = resolve(args.version).canonical
+    cases = None
+    if args.cases:
+        cases = [name.strip() for name in args.cases.split(",") if name.strip()]
+    try:
+        run = run_physics(
+            canonical,
+            fs_exe=args.fs_exe,
+            workroot=args.workroot,
+            cases=cases,
+            timeout_s=args.timeout,
+            smi_root=args.smi_root,
+        )
+    except PhysicsEnvironmentError as error:
+        print(f"physics run aborted: {error}", file=sys.stderr)
+        return 2
+    yaml_path, md_path = write_physics_report(run, args.report_dir, label=args.label)
+    counts = run.verdict_counts()
+    print(
+        f"FlightStream {run.version} ({run.fs_exe_name}): "
+        f"{counts['pass']} pass, {counts['warn']} warn, {counts['fail']} fail, "
+        f"{counts['no_reference']} without reference"
+    )
+    for result in run.results:
+        if result.error is not None:
+            print(f"{result.case_id} aborted: {result.error}", file=sys.stderr)
+    for line in run.solver_identity:
+        print(f"solver: {line}")
+    print(f"report: {yaml_path}")
+    print(f"report: {md_path}")
+    aborted = any(result.error is not None for result in run.results)
+    return 1 if counts["fail"] or aborted else 0
+
+
+def _cmd_drift(args: argparse.Namespace) -> int:
+    versions = [name.strip() for name in args.versions.split(",") if name.strip()]
+    if len(versions) != 2:
+        print(
+            "drift compares exactly two versions, baseline first "
+            "(for example --versions 26.100,26.120)",
+            file=sys.stderr,
+        )
+        return 2
+    fs_exes: dict[str, str] = {}
+    for item in args.fs_exe:
+        version, separator, path = item.partition("=")
+        if not separator or not path:
+            print(f"--fs-exe expects VERSION=PATH, got {item!r}", file=sys.stderr)
+            return 2
+        fs_exes[resolve(version.strip()).canonical] = path.strip()
+    canonicals = [resolve(version).canonical for version in versions]
+    missing = [canonical for canonical in canonicals if canonical not in fs_exes]
+    if missing:
+        print(
+            f"no executable given for {', '.join(missing)}; pass --fs-exe VERSION=PATH",
+            file=sys.stderr,
+        )
+        return 2
+    cases = None
+    if args.cases:
+        cases = [name.strip() for name in args.cases.split(",") if name.strip()]
+    try:
+        run = run_drift(
+            canonicals[0],
+            canonicals[1],
+            fs_exes=fs_exes,
+            workroot=args.workroot,
+            cases=cases,
+            timeout_s=args.timeout,
+            smi_root=args.smi_root,
+        )
+    except PhysicsEnvironmentError as error:
+        print(f"drift run aborted: {error}", file=sys.stderr)
+        return 2
+    yaml_path, md_path = write_drift_report(run, args.report_dir, label=args.label)
+    counts = run.verdict_counts()
+    print(
+        f"FlightStream {run.version_a} vs {run.version_b}: "
+        f"{counts['pass']} pass, {counts['warn']} warn, {counts['fail']} fail, "
+        f"{counts['no_reference']} without declared bands"
+    )
+    for result in run.results:
+        if result.error is not None:
+            print(f"{result.case_id} aborted: {result.error}", file=sys.stderr)
+    for line in run.solver_identity:
+        print(f"solver: {line}")
+    print(f"report: {yaml_path}")
+    print(f"report: {md_path}")
+    aborted = any(result.error is not None for result in run.results)
+    return 1 if counts["fail"] or aborted else 0
+
+
+def _cmd_update_reference(args: argparse.Namespace) -> int:
+    try:
+        path = update_reference(args.case, args.from_report, args.reason)
+    except ValueError as error:
+        print(f"reference not updated: {error}", file=sys.stderr)
+        return 2
+    print(f"reference written: {path}")
+    print("commit it alone: a reference update never shares a commit with code changes")
+    return 0
+
+
+def _cmd_apply_compat(args: argparse.Namespace) -> int:
+    promotions = apply_compat(args.report, repo_root=args.root)
+    if not promotions:
+        print("no verified or broken judgment in the report; nothing promoted")
+        return 0
+    for name, status, chapter in promotions:
+        print(f"{name}: {status} ({chapter})")
+    CommandRegistry.load.cache_clear()
+    CommandRegistry.load()
+    print(f"{len(promotions)} status(es) promoted; database reloaded and valid")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyflightstream/qa/compat.py

@@ -0,0 +1,285 @@
+"""Compat reports and status promotion (SAD Sections 3.2 and 11).
+
+Pipeline role: turns a probe run into committed evidence and the
+evidence into database statuses. :func:`write_compat_report` writes the
+machine-readable YAML report and its rendered Markdown table under
+``reports/compat/``, one evidence line per database command of the
+probed version. :func:`apply_compat` reads a committed report back and
+promotes ``documented`` statuses to ``verified`` or ``broken`` in the
+chapter YAML files, citing the report in each promoted entry; statuses
+are never hand-edited (CLAUDE.md invariant 3), and the schema rejects a
+``verified`` or ``broken`` entry without its report citation.
+"""
+
+from __future__ import annotations
+
+import datetime
+import re
+from importlib import resources
+from pathlib import Path
+
+import yaml
+
+from pyflightstream.commands import CommandEntry
+from pyflightstream.qa.probes import ProbeOutcome, ProbeRun
+
+COMPAT_SCHEMA = "pyflightstream-compat-report/1"
+
+
+def write_compat_report(
+    run: ProbeRun, out_dir: str | Path, *, date: str | None = None, label: str | None = None
+) -> tuple[Path, Path]:
+    """Write one probe run as a compat report pair (YAML plus Markdown).
+
+    The YAML file is the machine-readable evidence ``apply_compat``
+    reads; the Markdown file renders the same content as a table for
+    review. Both share the stem ``CMP-<version digits>_<date>`` plus
+    the optional label. Existing report files are never overwritten:
+    evidence supersedes evidence only through a new, dated report.
+
+    Parameters
+    ----------
+    run : ProbeRun
+        The probe run to record.
+    out_dir : str or Path
+        Target directory, normally ``reports/compat/``.
+    date : str, optional
+        ISO date stamped into the report; defaults to today.
+    label : str, optional
+        Stem suffix distinguishing several reports on the same day
+        (for example ``"full"`` after a pilot).
+
+    Returns
+    -------
+    tuple of Path
+        The YAML path and the Markdown path, in that order.
+
+    Raises
+    ------
+    FileExistsError
+        When a report with the same stem already exists.
+    """
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    date = date or datetime.date.today().isoformat()
+    stem = f"CMP-{run.version.replace('.', '')}_{date}"
+    if label:
+        stem += f"_{label}"
+    yaml_path = out_dir / f"{stem}.yaml"
+    md_path = out_dir / f"{stem}.md"
+    for path in (yaml_path, md_path):
+        if path.exists():
+            raise FileExistsError(
+                f"{path} already exists; compat reports are evidence and are never "
+                "overwritten. Remove the stale file deliberately or pick another date."
+            )
+    counts = run.outcome_counts()
+    document = {
+        "schema": COMPAT_SCHEMA,
+        "fs_version": run.version,
+        "date": date,
+        "package_version": run.package_version,
+        "fs_exe": run.fs_exe_name,
+        "executor": "LocalExecutor, -hidden --script (SRC-003 pp.279-280)",
+        "solver_identity": list(run.solver_identity),
+        "summary": counts,
+        "commands": {
+            result.command: {
+                "outcome": result.outcome.value,
+                "detail": result.detail,
+                "signals": {
+                    "sentinel_before": result.sentinel_before,
+                    "sentinel_after": result.sentinel_after,
+                    "effect": result.effect,
+                    "log_errors": list(result.log_errors),
+                },
+                "wall_time_s": None if result.wall_time_s is None else round(result.wall_time_s, 2),
+                "return_code": result.return_code,
+                "script_sha256": result.script_sha256,
+            }
+            for result in run.results
+        },
+    }
+    yaml_path.write_text(yaml.safe_dump(document, sort_keys=False, width=100), encoding="utf-8")
+    md_path.write_text(_render_markdown(run, date, counts), encoding="utf-8")
+    return yaml_path, md_path
+
+
+def _render_markdown(run: ProbeRun, date: str, counts: dict[str, int]) -> str:
+    """Render the human-readable side of the report."""
+    lines = [
+        f"# Compat report: FlightStream {run.version} ({date})",
+        "",
+        "Tier 2 command-validity evidence produced by the probe harness",
+        "(`pyfs-qa probe`); one evidence line per database command of this",
+        "version. Database statuses are promoted from this report only",
+        "through `pyfs-qa apply-compat`, never edited by hand (CLAUDE.md",
+        "invariant 3). Probe scripts and logs are local scratch; this",
+        "report is the committed evidence.",
+        "",
+        "## Setup",
+        "",
+        "| Item | Value |",
+        "|---|---|",
+        f"| Executable | {run.fs_exe_name} (local, `_private/exe/`, never committed) |",
+        "| Executor | LocalExecutor, `-hidden --script` (SRC-003 pp.279-280) |",
+        f"| Package | pyflightstream {run.package_version} |",
+        f"| Solver identity lines | {'; '.join(run.solver_identity) or 'none captured'} |",
+        "",
+        "## Summary",
+        "",
+        f"{counts['verified']} verified, {counts['broken']} broken, {counts['unprobed']} unprobed.",
+        "",
+        "## Evidence per command",
+        "",
+        "| Command | Outcome | Evidence |",
+        "|---|---|---|",
+    ]
+    for result in run.results:
+        detail = result.detail.replace("|", "\\|")
+        lines.append(f"| {result.command} | {result.outcome.value} | {detail} |")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def read_compat_report(path: str | Path) -> dict:
+    """Load and check a machine-readable compat report.
+
+    Parameters
+    ----------
+    path : str or Path
+        The report YAML file.
+
+    Returns
+    -------
+    dict
+        The parsed report document.
+
+    Raises
+    ------
+    ValueError
+        When the file does not carry the compat report schema marker.
+    """
+    document = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
+    if not isinstance(document, dict) or document.get("schema") != COMPAT_SCHEMA:
+        raise ValueError(
+            f"{path} is not a compat report (expected schema {COMPAT_SCHEMA!r}); "
+            "apply-compat only promotes statuses from probe evidence"
+        )
+    return document
+
+
+def apply_compat(
+    report_path: str | Path,
+    *,
+    repo_root: str | Path = ".",
+    commands_dir: str | Path | None = None,
+) -> list[tuple[str, str, str]]:
+    """Promote database statuses from a committed compat report.
+
+    Every command the report judged ``verified`` or ``broken`` gets its
+    version line in the chapter YAML rewritten to the new status with
+    the report cited in the ``report`` field; ``unprobed`` commands are
+    untouched. The edit is line-level on the flow-mapping version
+    lines, so chapter comments and layout survive; a version entry
+    spanning several lines is refused loudly rather than mangled.
+
+    Parameters
+    ----------
+    report_path : str or Path
+        The committed report YAML (``reports/compat/CMP-*.yaml``).
+    repo_root : str or Path
+        Repository root; the report citation written into the database
+        is the report path relative to it, in POSIX form.
+    commands_dir : str or Path, optional
+        Chapter YAML directory; defaults to the installed
+        ``pyflightstream.commands`` package data (the working tree in
+        an editable install). Tests point it at a copy.
+
+    Returns
+    -------
+    list of tuple
+        One ``(command, status, chapter file name)`` per promotion.
+
+    Raises
+    ------
+    ValueError
+        When the report is not a compat report, when a judged command
+        or its version line cannot be found, or when a rewritten entry
+        fails schema validation.
+    """
+    report = read_compat_report(report_path)
+    canonical = report["fs_version"]
+    citation = Path(report_path).resolve().relative_to(Path(repo_root).resolve()).as_posix()
+    if commands_dir is None:
+        commands_dir = Path(str(resources.files("pyflightstream.commands")))
+    commands_dir = Path(commands_dir)
+
+    targets = {
+        name: body
+        for name, body in report["commands"].items()
+        if body["outcome"] in (ProbeOutcome.VERIFIED.value, ProbeOutcome.BROKEN.value)
+    }
+    promotions: list[tuple[str, str, str]] = []
+    pending = dict(targets)
+    for chapter_path in sorted(commands_dir.glob("*.yaml")):
+        if chapter_path.name == "_meta.yaml":
+            continue
+        text = chapter_path.read_text(encoding="utf-8")
+        names = [name for name in yaml.safe_load(text) if name in pending]
+        if not names:
+            continue
+        for name in names:
+            body = pending.pop(name)
+            text = _rewrite_version_line(text, chapter_path.name, name, canonical, body, citation)
+        chapter_path.write_text(text, encoding="utf-8")
+        _validate_chapter(chapter_path, names)
+        promotions.extend((name, targets[name]["outcome"], chapter_path.name) for name in names)
+    if pending:
+        raise ValueError(
+            f"report judges {', '.join(sorted(pending))} but no chapter file defines "
+            "them; the report and the database have diverged"
+        )
+    return promotions
+
+
+def _rewrite_version_line(
+    text: str, chapter: str, name: str, canonical: str, body: dict, citation: str
+) -> str:
+    """Rewrite one command's version line to its promoted status."""
+    lines = text.splitlines()
+    start = None
+    end = len(lines)
+    for index, line in enumerate(lines):
+        if start is None:
+            if line.rstrip() == f"{name}:":
+                start = index
+        elif re.match(r"^[A-Za-z0-9_]+:", line):
+            end = index
+            break
+    if start is None:
+        raise ValueError(f"{chapter}: command block {name} not found")
+    pattern = re.compile(rf'^(\s+)"{re.escape(canonical)}":\s*\{{.*\}}\s*$')
+    for index in range(start, end):
+        match = pattern.match(lines[index])
+        if match is None:
+            continue
+        status = body["outcome"]
+        fields = f'status: {status}, report: "{citation}"'
+        if status == ProbeOutcome.BROKEN.value:
+            note = str(body.get("detail", "")).replace('"', "'").replace("\n", " ")[:140]
+            fields += f', note: "{note}"'
+        lines[index] = f'{match.group(1)}"{canonical}": {{{fields}}}'
+        return "\n".join(lines) + "\n"
+    raise ValueError(
+        f"{chapter}: no single-line version entry for {canonical!r} in {name}; "
+        "promote this entry manually reviewable or normalize the block first"
+    )
+
+
+def _validate_chapter(chapter_path: Path, names: list[str]) -> None:
+    """Re-validate the rewritten entries against the command schema."""
+    data = yaml.safe_load(chapter_path.read_text(encoding="utf-8"))
+    chapter = chapter_path.name.removesuffix(".yaml")
+    for name in names:
+        CommandEntry(name=name, chapter=chapter, **data[name])