agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/substitution_evidence.py

@@ -0,0 +1,199 @@
+"""Frozen decision-step measurements, separate from model-only profile totals."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+
+from agentloop.events import AgentEvent
+from agentloop.judgment_types import (
+    JudgeIdentity,
+    JudgeUsage,
+    canonical,
+    fingerprint,
+    finite,
+    is_hash,
+    text,
+)
+from agentloop.substitution_types import SUBSTITUTION_VERSION, TRIAL_KEY
+from agentloop.workflow_types import stage_summary, workflow_summary
+
+_STATES = {"completed", "unknown", "failed", "timed_out", "invalid_result", "disabled", "missing"}
+
+
+def _binding(trace):
+    events = []
+    for event in trace.events:
+        value = AgentEvent.from_dict(event.to_dict()).to_dict()
+        value["metadata"] = {
+            key: item
+            for key, item in value["metadata"].items()
+            if key not in {"otel_span_id", "otel_trace_id"}
+        }
+        events.append(value)
+    return fingerprint(
+        {
+            "run_id": trace.run_id,
+            "started_at": trace.started_at,
+            "ended_at": trace.ended_at,
+            "elapsed_ms": float(trace.elapsed_ms) if trace.elapsed_ms is not None else None,
+            "events": events,
+            "workflow": trace.metadata.get("agentloop.workflow"),
+            "trial_metadata": {
+                key: trace.metadata.get(key)
+                for key in (
+                    "plan_hash",
+                    "example_id",
+                    "task_id",
+                    "repetition",
+                    "condition",
+                    "synthetic",
+                    "success",
+                )
+            },
+        }
+    )
+
+
+def attach_trial_evidence(
+    trace,
+    *,
+    plan_hash,
+    example_id,
+    repetition,
+    implementation,
+    status,
+    latency_ms,
+    usage,
+    cost_ref,
+    input_hash,
+    output_hash,
+):
+    record = {
+        "schema_version": SUBSTITUTION_VERSION,
+        "run_id": trace.run_id,
+        "plan_hash": plan_hash,
+        "example_id": example_id,
+        "repetition": repetition,
+        "implementation": implementation,
+        "outcome": status,
+        "latency_ms": latency_ms,
+        "usage": asdict(usage),
+        "cost_ref": cost_ref,
+        "input_hash": input_hash,
+        "output_hash": output_hash,
+        "source_hash": _binding(trace),
+        "scope": "declared_decision_step",
+        "timing_basis": "callback_wall_time" if latency_ms is not None else "not_observed",
+    }
+    record["evidence_hash"] = fingerprint(record)
+    if TRIAL_KEY in trace.metadata:
+        raise ValueError("trial evidence cannot replace an existing receipt")
+    trace.metadata[TRIAL_KEY] = json.loads(canonical(record))
+    if read_trial_evidence(trace)["status"] != "valid":
+        trace.metadata.pop(TRIAL_KEY)
+        raise ValueError("invalid decision trial evidence")
+    return json.loads(canonical(record))
+
+
+def read_trial_evidence(trace):
+    if TRIAL_KEY not in trace.metadata:
+        return (
+            {"status": "invalid", "scope": "declared_decision_step"}
+            if trace.metadata.get("source") == "agentloop_substitution"
+            else None
+        )
+    empty = {"status": "invalid", "scope": "declared_decision_step"}
+    try:
+        record = json.loads(canonical(trace.metadata[TRIAL_KEY]))
+        if not isinstance(record, dict):
+            return empty
+        if record.get("schema_version") != SUBSTITUTION_VERSION:
+            return {**empty, "status": "unsupported"}
+        digest = record.pop("evidence_hash")
+        if (
+            digest != fingerprint(record)
+            or record["run_id"] != trace.run_id
+            or record["source_hash"] != _binding(trace)
+        ):
+            return empty
+        if record["scope"] != "declared_decision_step" or record["outcome"] not in _STATES:
+            return empty
+        for key in ("plan_hash", "input_hash"):
+            if not is_hash(record[key]):
+                return empty
+        if record["output_hash"] is not None and not is_hash(record["output_hash"]):
+            return empty
+        if (record["outcome"] == "completed") != (record["output_hash"] is not None):
+            return empty
+        text(record["example_id"], "example_id")
+        if type(record["repetition"]) is not int or record["repetition"] < 0:
+            return empty
+        declaration = record["implementation"]
+        text(declaration["name"], "implementation name")
+        JudgeIdentity(**declaration["identity"])
+        if declaration["kind"] not in {"model", "classifier", "rule", "tool", "external_service"}:
+            return empty
+        JudgeUsage(**record["usage"])
+        for key in ("plan_hash", "example_id", "repetition"):
+            if trace.metadata.get(key) != record[key]:
+                return empty
+        if (
+            trace.metadata.get("condition") != declaration["name"]
+            or type(trace.metadata.get("synthetic")) is not bool
+        ):
+            return empty
+        if trace.metadata.get("task_id") != record["example_id"]:
+            return empty
+        workflow = workflow_summary(trace.metadata)
+        if workflow is None or workflow.get("schema_status") != "supported":
+            return empty
+        failed = record["outcome"] in {"failed", "timed_out", "invalid_result"}
+        expected_state = (
+            "failed"
+            if failed
+            else ("unknown" if record["outcome"] in {"disabled", "missing"} else "completed")
+        )
+        if workflow["status"] != expected_state:
+            return empty
+        text(record["cost_ref"], "cost_ref", optional=True)
+        latency = record["latency_ms"]
+        if latency is None:
+            if (
+                record["outcome"] not in {"disabled", "missing"}
+                or record["timing_basis"] != "not_observed"
+                or trace.events
+            ):
+                return empty
+        elif (
+            not finite(latency)
+            or latency < 0
+            or latency != trace.elapsed_ms
+            or record["timing_basis"] != "callback_wall_time"
+            or len(trace.events) != 1
+            or trace.events[0].duration_ms != latency
+        ):
+            return empty
+        if latency is not None and trace.events[0].status != ("error" if failed else "ok"):
+            return empty
+        if latency is not None:
+            stage = stage_summary(trace.events[0])
+            if (
+                stage is None
+                or stage.get("schema_status") != "supported"
+                or stage.get("version") != declaration["identity"]["version"]
+                or stage.get("kind") != declaration["kind"]
+                or stage.get("outcome") != record["outcome"]
+                or stage.get("output_ref")
+                != (
+                    "sha256:" + record["output_hash"] if record["output_hash"] is not None else None
+                )
+            ):
+                return empty
+        if record["outcome"] == "disabled" and JudgeUsage(**record["usage"]) != JudgeUsage(
+            0, 0, 0, "not_dispatched", "not_dispatched"
+        ):
+            return empty
+        return {**record, "status": "valid", "evidence_hash": digest}
+    except (TypeError, ValueError, KeyError, AttributeError):
+        return empty

agentloop/substitution_export.py

@@ -0,0 +1,190 @@
+"""Export frozen decision trials without invoking an implementation or scorer."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from agentloop.markdown import markdown_heading, markdown_table_cell
+from agentloop.structured_quality import attach_quality_report
+from agentloop.studies import study_to_markdown, summarize_study
+from agentloop.substitution_reports import (
+    _copy,
+    index_trials,
+    load_bundle,
+    paired_assessments,
+    summarize_substitution_experiment,
+)
+from agentloop.tracer import AgentTrace
+
+
+def substitution_to_markdown(report):
+    lines = [
+        f"# Model substitution: {markdown_heading(report['name'])}",
+        "",
+        report["interpretation"],
+        "",
+        "Measurement scope: declared decision-step usage and callback latency. Reported/calculated usage is separate from model-only profile totals; unknown values are not zero.",
+        "",
+    ]
+    if report["synthetic"]:
+        lines.extend(
+            [
+                "Synthetic data: these results do not establish real-model quality, latency or savings.",
+                "",
+            ]
+        )
+    lines.extend(
+        [
+            "## Measurements",
+            "",
+            "| Implementation | Planned | Unavailable output rate | Observed latency ms | Observed cost USD | Cost basis |",
+            "| --- | ---: | ---: | ---: | ---: | --- |",
+        ]
+    )
+    for item in report["implementations"]:
+        cells = [
+            item["name"],
+            item["planned"],
+            item["unavailable_output_rate"],
+            item["measurements"]["latency_ms"]["observed_mean"],
+            item["measurements"]["cost_usd"]["observed_mean"],
+            json.dumps(item["cost_basis_counts"], sort_keys=True),
+        ]
+        lines.append(
+            "| "
+            + " | ".join(
+                markdown_table_cell("unavailable" if value is None else str(value))
+                for value in cells
+            )
+            + " |"
+        )
+    lines.extend(
+        [
+            "",
+            "## Independent quality and baseline agreement",
+            "",
+            "A full quality decision requires every planned example and repetition. Observed quality means use completely assessed examples only. Baseline agreement is a separate descriptive metric, not independent quality evidence.",
+            "",
+            "| Candidate | Complete examples | Baseline quality | Candidate quality | Quality preserved on examples | Baseline agreement |",
+            "| --- | ---: | ---: | ---: | --- | ---: |",
+        ]
+    )
+    for item in report["comparisons"]:
+        cells = [
+            item["condition"],
+            item["complete_quality_examples"],
+            item["observed_baseline_quality"],
+            item["observed_candidate_quality"],
+            item["quality_preserved_on_examples"],
+            item["baseline_agreement"]["observed_rate"],
+        ]
+        lines.append(
+            "| "
+            + " | ".join(
+                markdown_table_cell("unavailable" if value is None else str(value))
+                for value in cells
+            )
+            + " |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def export_substitution_studies(bundle, out, *, max_invocations=10000):
+    """Write native quality-bound traces and one paired study per candidate.
+
+    Failed/unknown outputs have real trial receipts and remain in each study.
+    Missing or ambiguous source records are not fabricated during export.
+    Baseline executions are shared controls across these separate comparisons.
+    """
+    owned, examples = load_bundle(bundle, max_invocations)
+    slots, extras = index_trials(owned, examples)
+    pairs, pair_extras = paired_assessments(owned, slots, examples)
+    if (
+        extras
+        or pair_extras
+        or any(item["trace"] is None for item in slots.values())
+        or any(item["quality"] is None for item in pairs)
+    ):
+        raise ValueError(
+            "study export requires unambiguous receipts and saved quality assessments; failed outputs remain valid study inputs"
+        )
+    root = Path(out).resolve()
+    documents, manifests = {}, []
+    for index, name in enumerate(
+        key for key in owned["implementations"] if key != owned["baseline"]
+    ):
+        directory = f"candidate-{index:03d}"
+        conditions = {"baseline": [], "candidate": []}
+        selected = [item for item in pairs if item["condition"] == name]
+        for row_index, pair in enumerate(selected):
+            for side, condition in (("baseline", owned["baseline"]), ("candidate", name)):
+                trial = slots[condition, pair["example_id"], pair["repetition"]]
+                trace = AgentTrace.from_dict(_copy(trial["trace"].to_dict()))
+                attach_quality_report(trace, pair["quality"], side=side)
+                filename = f"{side}-{row_index:06d}.json"
+                documents[f"{directory}/{filename}"] = trace.to_dict()
+                conditions[side].append(filename)
+        manifest = {
+            "schema_version": "1.0",
+            "name": owned["protocol"]["name"] + " / " + name,
+            "baseline": "baseline",
+            "conditions": conditions,
+            "pairing_keys": ["plan_hash", "example_id", "repetition"],
+        }
+        documents[f"{directory}/study.json"] = manifest
+        manifests.append(root / directory / "study.json")
+    comparison = summarize_substitution_experiment(owned, max_invocations=max_invocations)
+    documents["bundle.json"] = owned
+    documents["comparison.json"] = comparison
+    encoded = {
+        relative: json.dumps(value, indent=2, allow_nan=False) + "\n"
+        for relative, value in documents.items()
+    }
+    encoded["comparison.md"] = substitution_to_markdown(comparison)
+    for relative, content in encoded.items():
+        target = root / relative
+        if not target.resolve().is_relative_to(root):
+            raise ValueError("artifact path escapes the output directory")
+        if target.exists() and target.read_text(encoding="utf-8") != content:
+            raise FileExistsError(
+                f"refusing to replace a different experiment artifact: {relative}"
+            )
+    # Derived reports must also be protected. Their content is deterministic for
+    # these saved traces; checking after computation never reruns a scorer.
+    for manifest in manifests:
+        targets = [manifest.with_name(name) for name in ("study-report.json", "study-report.md")]
+        if any(not target.resolve().is_relative_to(root) for target in targets):
+            raise ValueError("study report path escapes the output directory")
+        if any(target.exists() for target in targets):
+            if not manifest.exists():
+                raise FileExistsError("orphaned study report would be overwritten")
+            existing = summarize_study(manifest)
+            expected = [
+                json.dumps(existing, indent=2, allow_nan=False) + "\n",
+                study_to_markdown(existing),
+            ]
+            for target, content in zip(targets, expected):
+                if target.exists() and target.read_text(encoding="utf-8") != content:
+                    raise FileExistsError("refusing to replace a different saved study report")
+    for relative, content in encoded.items():
+        target = root / relative
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if not target.exists():
+            target.write_text(content, encoding="utf-8")
+    reports = []
+    for manifest in manifests:
+        report = summarize_study(manifest)
+        derived = {
+            "study-report.json": json.dumps(report, indent=2, allow_nan=False) + "\n",
+            "study-report.md": study_to_markdown(report),
+        }
+        for name, content in derived.items():
+            target = manifest.with_name(name)
+            if target.exists() and target.read_text(encoding="utf-8") != content:
+                raise FileExistsError("refusing to replace a different saved study report")
+        for name, content in derived.items():
+            if not manifest.with_name(name).exists():
+                manifest.with_name(name).write_text(content, encoding="utf-8")
+        reports.append(report)
+    return {"directory": str(root), "studies": reports, "comparison": comparison}