runflect 0.1.0__py3-none-any.whl

runflect/__init__.py

@@ -0,0 +1,26 @@
+"""Runflect: turn agent execution into actionable evidence."""
+
+from .recorder import Run, Span, run
+from .policy import (
+    Approval,
+    ApprovalAuthority,
+    ApprovalRequest,
+    ApprovalRequiredError,
+    AuditWriteError,
+    InvalidApprovalError,
+    PolicyConfigurationError,
+    PolicyEvaluationError,
+    ToolDecision,
+    ToolDenied,
+    ToolGuard,
+    ToolPolicy,
+    policy_audit_context,
+)
+
+__all__ = [
+    "Run", "Span", "run", "Approval", "ApprovalAuthority", "ApprovalRequest",
+    "ApprovalRequiredError", "AuditWriteError", "InvalidApprovalError",
+    "PolicyConfigurationError", "PolicyEvaluationError", "ToolDecision",
+    "ToolDenied", "ToolGuard", "ToolPolicy", "policy_audit_context",
+]
+__version__ = "0.1.0"

runflect/__main__.py

@@ -0,0 +1,5 @@
+"""Support ``python -m runflect`` with the same behavior as the console script."""
+
+from .cli import main
+
+raise SystemExit(main())

runflect/bench/__init__.py

@@ -0,0 +1,4 @@
+"""Runflect Bench: local, deterministic checks of policy and recording behavior."""
+
+BENCHMARK_VERSION = "0.1.0"
+REPORT_SCHEMA = "runflect-bench-report/v1"

runflect/bench/fixtures.py

@@ -0,0 +1,33 @@
+"""Harmless deterministic tool fixtures used by the Bench suite."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+class BenchmarkToolError(RuntimeError):
+    """Expected, controlled tool failure used to test error propagation."""
+
+
+@dataclass
+class SideEffectCounter:
+    calls: int = 0
+
+
+def sync_fixture(counter: SideEffectCounter, behavior: str):
+    def tool(**_arguments: Any) -> str:
+        counter.calls += 1
+        if behavior == "raise":
+            raise BenchmarkToolError("controlled benchmark tool failure")
+        return "fixture-ok"
+    return tool
+
+
+def async_fixture(counter: SideEffectCounter, behavior: str):
+    async def tool(**_arguments: Any) -> str:
+        counter.calls += 1
+        if behavior == "raise":
+            raise BenchmarkToolError("controlled benchmark tool failure")
+        return "fixture-ok"
+    return tool

runflect/bench/metrics.py

@@ -0,0 +1,91 @@
+"""Precisely defined aggregate metrics for benchmark outcomes."""
+
+from __future__ import annotations
+
+import math
+from statistics import median
+from typing import Any, Iterable
+
+
+def _ratio(numerator: float, denominator: float) -> float | None:
+    return numerator / denominator if denominator else None
+
+
+def latency_summary(samples_ms: Iterable[float]) -> dict[str, Any]:
+    values = [float(value) for value in samples_ms]
+    if not values:
+        return {"sample_count": 0, "median_ms": None, "p95_ms": None, "samples_ms": []}
+    if any(not math.isfinite(value) or value < 0 for value in values):
+        raise ValueError("latency samples must be finite and non-negative")
+    ordered = sorted(values)
+    p95 = ordered[math.ceil(0.95 * len(ordered)) - 1] if len(ordered) >= 20 else None
+    return {"sample_count": len(values), "median_ms": median(values), "p95_ms": p95, "samples_ms": values}
+
+
+def calculate_metrics(results: list[dict[str, Any]], mode: str) -> dict[str, Any]:
+    """Aggregate scenario rows. Counts are scenario-action units unless specified."""
+    selected = [row for row in results if row["configuration"] == mode]
+    if not selected:
+        raise ValueError(f"no results for configuration {mode}")
+    total_rows = len(selected)
+    passed = sum(bool(row["scenario_passed"]) for row in selected)
+    enforce_rows = [row for row in selected if mode != "baseline"]
+    decision_n = len(enforce_rows)
+    decision_correct = sum(row["actual_decision"] == row["expected_decision"] for row in enforce_rows)
+    unauthorized_actions = sum(
+        row["attempts"] for row in selected if not row["expected_execution"]
+    )
+    unauthorized_executions = sum(
+        min(row["actual_executions"], row["attempts"])
+        for row in selected if not row["expected_execution"]
+    )
+    blocked_actions = unauthorized_actions - unauthorized_executions
+    permitted_actions = sum(row["attempts"] for row in selected if row["expected_execution"])
+    false_blocks = sum(
+        max(0, row["attempts"] - min(row["actual_executions"], row["attempts"]))
+        for row in selected if row["expected_execution"]
+    )
+    approved_policy_rows = [row for row in enforce_rows if row["category"] == "approval"]
+    trace_rows = [row for row in selected if row.get("trace_complete") is not None]
+    tool_failure_count = sum(row["tool_failures"] for row in selected)
+    expected_tool_failure_rows = [row for row in selected if row["behavior"] == "raise"]
+    expected_tool_failure_handled = sum(row["expected_tool_failure_handled"] for row in expected_tool_failure_rows)
+    return {
+        "scenario_count": total_rows,
+        "scenario_pass_rate": {"numerator": passed, "denominator": total_rows, "value": passed / total_rows},
+        "policy_decision_accuracy": {
+            "numerator": decision_correct, "denominator": decision_n,
+            "value": _ratio(decision_correct, decision_n),
+            "scope": "policy and recorded configurations; baseline has no decision",
+        },
+        "unauthorized_execution_rate": {
+            "numerator": unauthorized_executions, "denominator": unauthorized_actions,
+            "value": _ratio(unauthorized_executions, unauthorized_actions),
+        },
+        "correctly_blocked_action_rate": {
+            "numerator": blocked_actions, "denominator": unauthorized_actions,
+            "value": _ratio(blocked_actions, unauthorized_actions),
+        },
+        "false_block_rate": {
+            "numerator": false_blocks, "denominator": permitted_actions,
+            "value": _ratio(false_blocks, permitted_actions),
+        },
+        "approval_enforcement_accuracy": {
+            "numerator": sum(row["scenario_passed"] for row in approved_policy_rows),
+            "denominator": len(approved_policy_rows),
+            "value": _ratio(sum(row["scenario_passed"] for row in approved_policy_rows), len(approved_policy_rows)),
+            "scope": "approval category under policy-enforced configurations",
+        },
+        "trace_completeness": {
+            "numerator": sum(row["trace_complete"] is True for row in trace_rows),
+            "denominator": len(trace_rows),
+            "value": _ratio(sum(row["trace_complete"] is True for row in trace_rows), len(trace_rows)),
+            "scope": "recorded configuration only",
+        },
+        "tool_failures_observed": tool_failure_count,
+        "expected_tool_failure_handling_rate": {
+            "numerator": expected_tool_failure_handled,
+            "denominator": len(expected_tool_failure_rows),
+            "value": _ratio(expected_tool_failure_handled, len(expected_tool_failure_rows)),
+        },
+    }

runflect/bench/regression.py

@@ -0,0 +1,320 @@
+"""Validated Bench baseline snapshots and scenario-level regression comparison."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import re
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from . import REPORT_SCHEMA
+
+BASELINE_SCHEMA = "runflect-bench-baseline/v1"
+COMPARISON_SCHEMA = "runflect-bench-comparison/v1"
+_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
+_LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.: -]{0,119}$")
+_MODES = ("policy_enforcement", "recording_and_enforcement")
+_CONFIGURATIONS = ("baseline", *_MODES)
+
+
+class BenchReportError(ValueError):
+    """A Bench report or baseline is malformed or incompatible."""
+
+
+class BenchRegressionError(OSError):
+    """A baseline or comparison report could not be read or written."""
+
+
+def _canonical(value: Any) -> str:
+    try:
+        return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"), allow_nan=False)
+    except (TypeError, ValueError) as error:
+        raise BenchReportError("report must contain finite JSON values") from error
+
+
+def _read_json(path: str | Path, label: str) -> Any:
+    try:
+        source = Path(path)
+        if source.stat().st_size > 50 * 1024 * 1024:
+            raise BenchReportError(f"{label} exceeds the 50 MiB input limit")
+        return json.loads(source.read_text(encoding="utf-8"))
+    except BenchReportError:
+        raise
+    except FileNotFoundError as error:
+        raise BenchReportError(f"{label} file was not found") from error
+    except json.JSONDecodeError as error:
+        raise BenchReportError(f"{label} file is malformed JSON") from error
+    except UnicodeError as error:
+        raise BenchReportError(f"{label} file is not UTF-8 JSON") from error
+    except OSError as error:
+        raise BenchRegressionError(f"{label} file could not be read") from error
+
+
+def _sample_stat(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {"sample_count": 0, "median_ms": None, "p95_ms": None}
+    count = value.get("sample_count")
+    count = count if isinstance(count, int) and not isinstance(count, bool) and 0 <= count <= 10000 else 0
+    result = {"sample_count": count}
+    for key in ("median_ms", "p95_ms"):
+        number = value.get(key)
+        result[key] = number if isinstance(number, (int, float)) and not isinstance(number, bool) and math.isfinite(number) and number >= 0 else None
+    return result
+
+
+def validate_report(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict) or value.get("schema") != REPORT_SCHEMA:
+        raise BenchReportError("unsupported Bench report schema")
+    benchmark_version = value.get("benchmark_version")
+    suite = value.get("suite")
+    results = value.get("scenario_results")
+    if (not isinstance(benchmark_version, str) or not _ID.fullmatch(benchmark_version)
+            or not isinstance(suite, dict) or not isinstance(results, list)):
+        raise BenchReportError("Bench report metadata is incomplete")
+    suite_id, suite_version = suite.get("id"), suite.get("version")
+    digest, scenario_ids = suite.get("digest"), suite.get("scenario_ids")
+    if (not isinstance(suite_id, str) or not _ID.fullmatch(suite_id)
+            or not isinstance(suite_version, str) or not _ID.fullmatch(suite_version)
+            or not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest)
+            or not isinstance(scenario_ids, list) or not scenario_ids or len(scenario_ids) > 500
+            or any(not isinstance(item, str) or not _ID.fullmatch(item) for item in scenario_ids)
+            or len(scenario_ids) != len(set(scenario_ids))):
+        raise BenchReportError("Bench suite identity is invalid")
+    outcomes = {mode: {} for mode in _CONFIGURATIONS}
+    for row in results:
+        if not isinstance(row, dict):
+            raise BenchReportError("Bench scenario result is invalid")
+        scenario_id, mode, passed = row.get("scenario_id"), row.get("configuration"), row.get("scenario_passed")
+        if (not isinstance(scenario_id, str) or scenario_id not in scenario_ids
+                or not isinstance(mode, str) or mode not in outcomes or type(passed) is not bool):
+            raise BenchReportError("Bench scenario result contains unsupported identity or outcome")
+        if scenario_id in outcomes[mode]:
+            raise BenchReportError("Bench report contains duplicate scenario results")
+        outcomes[mode][scenario_id] = passed
+    required = set(scenario_ids)
+    if any(set(outcomes[mode]) != required for mode in _CONFIGURATIONS):
+        raise BenchReportError("Bench report is missing one or more configured scenario outcomes")
+    run = value.get("run") if isinstance(value.get("run"), dict) else {}
+    run_id = run.get("id")
+    if not isinstance(run_id, str):
+        run_id = None
+    else:
+        try:
+            run_id = str(uuid.UUID(run_id))
+        except ValueError:
+            run_id = None
+    performance = value.get("performance") if isinstance(value.get("performance"), dict) else {}
+    statistics = performance.get("statistics") if isinstance(performance.get("statistics"), dict) else {}
+    return {
+        "benchmark_version": benchmark_version,
+        "suite": {"id": suite_id, "version": suite_version, "digest": digest, "scenario_ids": list(scenario_ids)},
+        "run_id": run_id,
+        "report_digest": hashlib.sha256(_canonical(value).encode("ascii")).hexdigest(),
+        "outcomes": outcomes,
+        "performance": {mode: _sample_stat(statistics.get(mode)) for mode in _CONFIGURATIONS},
+    }
+
+
+def load_report(path: str | Path) -> tuple[dict[str, Any], dict[str, Any]]:
+    document = _read_json(path, "Bench report")
+    return document, validate_report(document)
+
+
+def create_baseline(report: dict[str, Any], *, label: str | None = None) -> dict[str, Any]:
+    validated = validate_report(report)
+    if label is not None and (not isinstance(label, str) or not _LABEL.fullmatch(label.strip())):
+        raise BenchReportError("baseline label must contain 1 to 120 characters")
+    return {
+        "schema": BASELINE_SCHEMA,
+        "baseline_id": str(uuid.uuid4()),
+        "label": label.strip() if label else "baseline",
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "benchmark_version": validated["benchmark_version"],
+        "suite": validated["suite"],
+        "source_run_id": validated["run_id"],
+        "source_report_digest": validated["report_digest"],
+        "outcomes": validated["outcomes"],
+        "performance": validated["performance"],
+    }
+
+
+def validate_baseline(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict) or value.get("schema") != BASELINE_SCHEMA:
+        raise BenchReportError("unsupported Bench baseline schema")
+    required = {"baseline_id", "benchmark_version", "suite", "outcomes", "performance", "source_report_digest"}
+    if not required.issubset(value):
+        raise BenchReportError("Bench baseline metadata is incomplete")
+    try:
+        uuid.UUID(value["baseline_id"])
+    except (ValueError, TypeError, AttributeError):
+        raise BenchReportError("Bench baseline identifier is invalid") from None
+    if not isinstance(value["source_report_digest"], str) or not re.fullmatch(r"[a-f0-9]{64}", value["source_report_digest"]):
+        raise BenchReportError("Bench baseline digest is invalid")
+    label = value.get("label", "baseline")
+    if not isinstance(label, str) or not _LABEL.fullmatch(label):
+        raise BenchReportError("Bench baseline label is invalid")
+    source_run_id = value.get("source_run_id")
+    if source_run_id is not None:
+        try:
+            uuid.UUID(source_run_id)
+        except (ValueError, TypeError, AttributeError):
+            raise BenchReportError("Bench baseline source run reference is invalid") from None
+    suite = value["suite"]
+    if not isinstance(suite, dict):
+        raise BenchReportError("Bench baseline suite metadata is invalid")
+    outcomes = value.get("outcomes")
+    if not isinstance(outcomes, dict) or set(outcomes) != set(_CONFIGURATIONS):
+        raise BenchReportError("Bench baseline outcomes are invalid")
+    for mode in _CONFIGURATIONS:
+        scenarios = outcomes[mode]
+        if not isinstance(scenarios, dict) or any(not isinstance(key, str) or type(passed) is not bool
+                                                  for key, passed in scenarios.items()):
+            raise BenchReportError("Bench baseline scenario outcomes are invalid")
+    performance = value.get("performance")
+    if not isinstance(performance, dict) or set(performance) != set(_CONFIGURATIONS):
+        raise BenchReportError("Bench baseline performance metadata is invalid")
+    for mode in _CONFIGURATIONS:
+        stat = performance[mode]
+        if not isinstance(stat, dict):
+            raise BenchReportError("Bench baseline performance metadata is invalid")
+    baseline_report = {
+        "schema": REPORT_SCHEMA,
+        "benchmark_version": value.get("benchmark_version"),
+        "suite": suite,
+        "scenario_results": [
+            {"scenario_id": scenario_id, "configuration": mode, "scenario_passed": passed}
+            for mode, scenarios in value["outcomes"].items()
+            for scenario_id, passed in scenarios.items()
+        ],
+        "performance": {"statistics": performance},
+    }
+    normalized = validate_report(baseline_report)
+    return normalized
+
+
+def load_baseline(path: str | Path) -> tuple[dict[str, Any], dict[str, Any]]:
+    document = _read_json(path, "Bench baseline")
+    return document, validate_baseline(document)
+
+
+def compare(baseline_doc: dict[str, Any], baseline: dict[str, Any],
+            current_doc: dict[str, Any], current: dict[str, Any], *,
+            max_regressions: int = 0, max_pass_rate_drop: float | None = None,
+            max_p95_regression_percent: float | None = None) -> dict[str, Any]:
+    if isinstance(max_regressions, bool) or not isinstance(max_regressions, int) or max_regressions < 0:
+        raise BenchReportError("max_regressions must be a non-negative integer")
+    for key, threshold in (("max_pass_rate_drop", max_pass_rate_drop),
+                           ("max_p95_regression_percent", max_p95_regression_percent)):
+        maximum = 1 if key == "max_pass_rate_drop" else 1000
+        if threshold is not None and (isinstance(threshold, bool) or not isinstance(threshold, (int, float))
+                                      or not math.isfinite(threshold) or threshold < 0 or threshold > maximum):
+            raise BenchReportError(f"{key} must be a finite non-negative number")
+    if isinstance(max_regressions, bool) or not isinstance(max_regressions, int) or not 0 <= max_regressions <= 1500:
+        raise BenchReportError("max_regressions must be between 0 and 1500")
+    if baseline["benchmark_version"] != current["benchmark_version"] or baseline["suite"] != current["suite"]:
+        raise BenchReportError("Bench reports are incompatible; benchmark and suite identity, version, digest, and scenario IDs must match")
+
+    config_results = {}
+    regressions = []
+    improvements = []
+    for mode in _MODES:
+        base = baseline["outcomes"][mode]
+        now = current["outcomes"][mode]
+        failures = []
+        better = []
+        for scenario_id in baseline["suite"]["scenario_ids"]:
+            if base[scenario_id] and not now[scenario_id]:
+                failures.append(scenario_id)
+                regressions.append({"configuration": mode, "scenario_id": scenario_id})
+            elif not base[scenario_id] and now[scenario_id]:
+                better.append(scenario_id)
+                improvements.append({"configuration": mode, "scenario_id": scenario_id})
+        base_rate = sum(base.values()) / len(base) if base else None
+        current_rate = sum(now.values()) / len(now) if now else None
+        config_results[mode] = {
+            "baseline_pass_rate": base_rate,
+            "current_pass_rate": current_rate,
+            "pass_rate_delta": current_rate - base_rate if current_rate is not None and base_rate is not None else None,
+            "regression_count": len(failures), "regressed_scenarios": failures,
+            "improved_scenarios": better,
+        }
+    performance_result: dict[str, Any] = {"status": "not_requested"}
+    performance_inconclusive = False
+    performance_failed = False
+    if max_p95_regression_percent is not None:
+        performance_result = {"status": "evaluated", "threshold_percent": max_p95_regression_percent, "modes": {}}
+        for mode in _MODES:
+            before = baseline["performance"][mode]
+            after = current["performance"][mode]
+            if (before["sample_count"] < 20 or after["sample_count"] < 20
+                    or before["p95_ms"] is None or after["p95_ms"] is None or before["p95_ms"] == 0):
+                performance_inconclusive = True
+                performance_result["status"] = "inconclusive"
+                performance_result["modes"][mode] = {
+                    "status": "insufficient_samples", "minimum_sample_count": 20,
+                    "baseline_sample_count": before["sample_count"], "current_sample_count": after["sample_count"],
+                    "baseline_p95_ms": before["p95_ms"], "current_p95_ms": after["p95_ms"],
+                }
+                continue
+            delta = (after["p95_ms"] - before["p95_ms"]) / before["p95_ms"] * 100
+            failed = delta > max_p95_regression_percent
+            performance_failed = performance_failed or failed
+            performance_result["modes"][mode] = {
+                "status": "failed" if failed else "passed", "baseline_sample_count": before["sample_count"],
+                "current_sample_count": after["sample_count"], "baseline_p95_ms": before["p95_ms"],
+                "current_p95_ms": after["p95_ms"], "p95_delta_percent": delta,
+            }
+
+    pass_rate_failed = bool(max_pass_rate_drop is not None and any(
+        -item["pass_rate_delta"] > max_pass_rate_drop for item in config_results.values()
+        if item["pass_rate_delta"] is not None
+    ))
+    functional_failed = len(regressions) > max_regressions or pass_rate_failed
+    status = "failed" if functional_failed or performance_failed else (
+        "inconclusive" if performance_inconclusive else "passed"
+    )
+    return {
+        "schema": COMPARISON_SCHEMA,
+        "comparison_id": str(uuid.uuid4()),
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "status": status,
+        "baseline": {"id": baseline_doc["baseline_id"], "label": baseline_doc.get("label", "baseline"),
+                     "source_run_id": baseline_doc.get("source_run_id"), "report_digest": baseline_doc["source_report_digest"]},
+        "current": {"run_id": current["run_id"], "report_digest": current["report_digest"]},
+        "suite": {"id": current["suite"]["id"], "version": current["suite"]["version"],
+                  "digest": current["suite"]["digest"], "scenario_count": len(current["suite"]["scenario_ids"])},
+        "thresholds": {"max_regressions": max_regressions, "max_pass_rate_drop": max_pass_rate_drop,
+                        "max_p95_regression_percent": max_p95_regression_percent},
+        "summary": {"regression_count": len(regressions), "improvement_count": len(improvements),
+                    "configuration_results": config_results},
+        "regressions": regressions,
+        "improvements": improvements,
+        "performance": performance_result,
+        "limitations": [
+            "This compares deterministic Bench scenario outcomes and configured local timing observations; it does not measure general agent quality.",
+            "P95 thresholds require at least 20 samples in both reports and are host-specific observations, not a statistical significance test.",
+            "Reports are comparable only when benchmark version, suite ID/version/digest, and scenario IDs match exactly.",
+        ],
+    }
+
+
+def write_json(value: dict[str, Any], path: str | Path) -> None:
+    target = Path(path).expanduser()
+    try:
+        target.parent.mkdir(parents=True, exist_ok=True)
+        temporary = target.with_name(f".{target.name}.{uuid.uuid4().hex}.tmp")
+        temporary.write_text(json.dumps(value, ensure_ascii=True, sort_keys=True, indent=2, allow_nan=False) + "\n", encoding="utf-8")
+        temporary.replace(target)
+    except OSError as error:
+        raise BenchRegressionError("comparison artifact could not be written") from error
+
+
+def save_baseline(report_path: str | Path, output_path: str | Path, *, label: str | None = None) -> dict[str, Any]:
+    document, _ = load_report(report_path)
+    baseline = create_baseline(document, label=label)
+    write_json(baseline, output_path)
+    return baseline

runflect/bench/report.py

@@ -0,0 +1,76 @@
+"""Stable JSON report construction and serialization for Runflect Bench."""
+
+from __future__ import annotations
+
+import importlib.metadata
+import json
+import platform
+import sys
+import uuid
+from datetime import datetime, timezone
+from typing import Any
+
+from .. import __version__
+from . import BENCHMARK_VERSION, REPORT_SCHEMA
+from .schema import BenchSuite
+
+
+def _package_version(name: str) -> str | None:
+    try:
+        return importlib.metadata.version(name)
+    except importlib.metadata.PackageNotFoundError:
+        return None
+
+
+def build_report(suite: BenchSuite, *, results: list[dict[str, Any]],
+                 metrics: dict[str, Any], performance: dict[str, Any],
+                 policy_summary: dict[str, Any]) -> dict[str, Any]:
+    """Assemble the versioned, argument-free benchmark report."""
+    enforced_pass = all(row["scenario_passed"] for row in results
+                        if row["configuration"] != "baseline")
+    return {
+        "schema": REPORT_SCHEMA,
+        "status": "passed" if enforced_pass else "scenario_failures",
+        "enforcement_passed": enforced_pass,
+        "benchmark_version": BENCHMARK_VERSION,
+        "suite": {"id": suite.suite_id, "version": suite.suite_version,
+                  "digest": suite.digest, "source": suite.source,
+                  "scenario_ids": [item.scenario_id for item in suite.scenarios]},
+        "run": {"id": str(uuid.uuid4()), "timestamp_utc": datetime.now(timezone.utc).isoformat(),
+                "random_seed": None, "runtime": {"python": sys.version.split()[0],
+                "platform": platform.platform(), "runflect": __version__,
+                "langgraph": _package_version("langgraph"),
+                "langchain_core": _package_version("langchain-core")}},
+        "configurations": {
+            "baseline": {"label": "baseline", "includes": [],
+                         "excludes": ["policy enforcement", "policy audit", "trace recording"]},
+            "policy_enforcement": {"label": "policy_enforcement",
+                                   "includes": ["ToolGuard policy evaluation", "durable policy audit"],
+                                   "excludes": ["Run/Span trace recording"]},
+            "recording_and_enforcement": {"label": "recording_and_enforcement",
+                                           "includes": ["ToolGuard policy evaluation", "durable policy audit",
+                                                        "Run/Span trace recording"]},
+        },
+        "policies": policy_summary,
+        "scenario_results": results,
+        "metrics": metrics,
+        "performance": performance,
+        "measurement_limitations": [
+            "Fixtures are deterministic and local; they do not call a model or external service.",
+            "The pure scenario evaluator supplies an off-path reference decision; baseline tool calls themselves bypass policy, and reference evaluation is excluded from latency measurements.",
+            "Prompt-injection scenarios are strings passed to controlled tools, not evidence of real-world injection resistance.",
+            "The baseline intentionally bypasses policy and may execute harmless simulated unauthorized actions.",
+            "Policy enforcement includes durable SQLite policy-event writes; recording_and_enforcement adds Run/Span writes.",
+            "Database setup and policy revision import are excluded from timed observations; per-call SQLite writes are included.",
+            "Latency results depend on this host, Python, SQLite, filesystem, and concurrent system load; no performance threshold is asserted.",
+            "No memory measurement is reported because this harness does not collect a reliable comparable metric.",
+            "A local digest and report aid reproducibility but do not establish independent integrity or trustworthiness.",
+        ],
+    }
+
+
+def serialize_report(report: dict[str, Any], *, pretty: bool = False) -> str:
+    """Serialize reports deterministically, with compact and readable forms."""
+    if pretty:
+        return json.dumps(report, ensure_ascii=True, sort_keys=True, indent=2)
+    return json.dumps(report, ensure_ascii=True, sort_keys=True, separators=(",", ":"))