devagent-physical-engine 0.10.0__py3-none-any.whl

devagent_physical_engine/qualification_cli.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import argparse
+import json
+
+from .qualification_harness import (
+    QualificationError,
+    QualificationStore,
+    base_case,
+    compare_summaries,
+    generate_fuzz_cases,
+    load_scenario,
+    load_scenario_suite,
+    load_summary,
+    replay_case,
+    run_cases,
+    variation_space_size,
+)
+
+
+DEFAULT_QUALIFICATION_ROOT = "~/.devagent/qualification"
+
+
+def add_qualification_parsers(sub: argparse._SubParsersAction) -> None:
+    test = sub.add_parser(
+        "test",
+        help="Run one deterministic qualification scenario and record evidence",
+    )
+    test.add_argument("scenario")
+    test.add_argument("--output-dir", default=DEFAULT_QUALIFICATION_ROOT)
+
+    suite = sub.add_parser(
+        "test-suite",
+        help="Run all YAML/JSON scenarios in a directory",
+    )
+    suite.add_argument("scenario_dir")
+    suite.add_argument("--output-dir", default=DEFAULT_QUALIFICATION_ROOT)
+
+    fuzz = sub.add_parser(
+        "fuzz",
+        help="Run seeded dynamic deterministic-model qualification cases",
+    )
+    fuzz.add_argument("scenario")
+    fuzz.add_argument("--runs", type=int, default=1000)
+    fuzz.add_argument("--seed", type=int, default=0)
+    fuzz.add_argument("--output-dir", default=DEFAULT_QUALIFICATION_ROOT)
+
+    replay = sub.add_parser(
+        "replay",
+        help="Re-run an exact recorded deterministic qualification case",
+    )
+    replay.add_argument("case_id")
+    replay.add_argument("--run", default=None)
+    replay.add_argument("--store-root", default=DEFAULT_QUALIFICATION_ROOT)
+    replay.add_argument(
+        "--visual",
+        action="store_true",
+        help="Request visual replay; v0.6 fails closed because deterministic cases do not encode Gazebo geometry",
+    )
+
+    compare = sub.add_parser(
+        "compare",
+        help="Compare two qualification summaries on the same case corpus",
+    )
+    compare.add_argument("baseline")
+    compare.add_argument("candidate")
+
+
+def _emit_run(summary, run_dir, *, extra: dict | None = None) -> None:
+    payload = {
+        "summary": summary.to_dict(),
+        "run_dir": str(run_dir),
+    }
+    if extra:
+        payload.update(extra)
+    print(json.dumps(payload, indent=2))
+
+
+def _test(args: argparse.Namespace) -> int:
+    scenario = load_scenario(args.scenario)
+    cases = (base_case(scenario),)
+    results, summary = run_cases(cases, mode="test", seed=0)
+    run_dir = QualificationStore(args.output_dir).write_run(
+        scenarios=(scenario,),
+        results=results,
+        summary=summary,
+    )
+    _emit_run(summary, run_dir, extra={"result": results[0].to_dict()})
+    return 0 if summary.qualified else 10
+
+
+def _test_suite(args: argparse.Namespace) -> int:
+    scenarios = load_scenario_suite(args.scenario_dir)
+    cases = tuple(
+        base_case(scenario, index=index, seed=0)
+        for index, scenario in enumerate(scenarios, start=1)
+    )
+    results, summary = run_cases(cases, mode="test-suite", seed=0)
+    run_dir = QualificationStore(args.output_dir).write_run(
+        scenarios=scenarios,
+        results=results,
+        summary=summary,
+    )
+    _emit_run(summary, run_dir)
+    return 0 if summary.qualified else 10
+
+
+def _fuzz(args: argparse.Namespace) -> int:
+    scenario = load_scenario(args.scenario)
+    cases = generate_fuzz_cases(
+        scenario,
+        runs=args.runs,
+        seed=args.seed,
+    )
+    results, summary = run_cases(cases, mode="fuzz", seed=args.seed)
+    run_dir = QualificationStore(args.output_dir).write_run(
+        scenarios=(scenario,),
+        results=results,
+        summary=summary,
+    )
+    _emit_run(
+        summary,
+        run_dir,
+        extra={
+            "variation_space_size": variation_space_size(scenario),
+            "note": (
+                "deterministic_model qualification only; "
+                "physical_qualification remains false"
+            ),
+        },
+    )
+    return 0 if summary.qualified else 10
+
+
+def _replay(args: argparse.Namespace) -> int:
+    if args.visual:
+        print(
+            json.dumps(
+                {
+                    "passed": False,
+                    "failure_code": "visual_replay_not_supported_v06",
+                    "detail": (
+                        "Recorded v0.6 qualification cases exercise the deterministic "
+                        "model backend and do not encode Gazebo world geometry. "
+                        "Exact deterministic replay is supported without --visual."
+                    ),
+                },
+                indent=2,
+            )
+        )
+        return 12
+    payload = replay_case(
+        args.case_id,
+        run_ref=args.run,
+        store_root=args.store_root,
+    )
+    print(json.dumps(payload, indent=2))
+    return 0 if payload["reproduced"] else 11
+
+
+def _compare(args: argparse.Namespace) -> int:
+    payload = compare_summaries(
+        load_summary(args.baseline),
+        load_summary(args.candidate),
+    )
+    print(json.dumps(payload, indent=2))
+    if not payload["comparable"]:
+        return 14
+    return 0 if payload["passed"] else 13
+
+
+def run_qualification_command(args: argparse.Namespace) -> int | None:
+    try:
+        if args.command == "test":
+            return _test(args)
+        if args.command == "test-suite":
+            return _test_suite(args)
+        if args.command == "fuzz":
+            return _fuzz(args)
+        if args.command == "replay":
+            return _replay(args)
+        if args.command == "compare":
+            return _compare(args)
+    except QualificationError as exc:
+        print(
+            json.dumps(
+                {
+                    "passed": False,
+                    "failure_code": str(exc),
+                },
+                indent=2,
+            )
+        )
+        return 10
+    return None