agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/experiments.py

@@ -0,0 +1,392 @@
+"""Bounded local candidate runs using native traces, budgets and interventions."""
+
+from __future__ import annotations
+
+import inspect
+from fractions import Fraction
+from pathlib import Path
+from time import monotonic, perf_counter
+
+from agentloop.ablation_protocol import timestamp
+from agentloop.budget_types import DispatchOptions
+from agentloop.budgets import BudgetLimits, budget_policy
+from agentloop.events import utc_now_iso
+from agentloop.experiment_artifacts import journal_lock, local_path, read_json, write_once
+from agentloop.experiment_types import (
+    EXPERIMENT_KEY,
+    ExperimentCase,
+    ExperimentPlan,
+    ExperimentRequest,
+    ExperimentResult,
+    ExperimentRunner,
+    canonical,
+    fingerprint,
+)
+from agentloop.harness import Harness, HarnessConfig, HarnessControlError
+from agentloop.interventions import build_intervention
+from agentloop.quality import build_quality_report
+from agentloop.replay import ReplayGates
+from agentloop.tracer import AgentTrace, bind_trace_context
+from agentloop.workflows import WorkflowInfo, record_operation, workflow_metadata
+
+
+def slot_id(experiment_id, case_id, candidate_id):
+    return fingerprint([experiment_id, case_id, candidate_id])
+
+
+class ExperimentSession:
+    """Resume finished slots idempotently; never repeat an uncertain attempt.
+
+    The root is a local evidence journal, not a workflow engine. Callback I/O
+    deadlines, provider limits and any side effects remain host responsibilities.
+    """
+
+    def __init__(self, plan, *, cases, runners, root):
+        if (
+            type(plan) is not ExperimentPlan
+            or any(type(case) is not ExperimentCase for case in cases)
+            or any(type(runner) is not ExperimentRunner for runner in runners)
+        ):
+            raise ValueError("typed experiment plan, cases and runner bindings required")
+        self._plan, self._spec = plan, plan.to_dict()
+        self._cases = {case.case_id: case for case in cases}
+        self._runners = {runner.candidate_id: runner for runner in runners}
+        if [case.declaration() for case in cases] != self._spec["cases"] or [
+            runner.declaration() for runner in runners
+        ] != self._spec["runners"]:
+            raise ValueError("experiment bindings differ from the frozen plan")
+        self.root = Path(root).resolve()
+        with journal_lock(self.root):
+            write_once(
+                self.root,
+                "experiment.json",
+                {"experiment_id": plan.experiment_id, "specification": self._spec},
+            )
+            for case in cases:
+                data = case.payload()
+                write_once(
+                    self.root, f"baselines/{fingerprint(case.case_id)}.json", data["baseline"]
+                )
+                write_once(
+                    self.root, f"predictions/{fingerprint(case.case_id)}.json", data["diagnosis"]
+                )
+
+    @classmethod
+    def resume(cls, root, *, cases, runners):
+        saved = read_json(local_path(root, "experiment.json"))
+        spec = saved["specification"]
+        from agentloop.experiment_types import ExperimentBudget
+
+        plan = ExperimentPlan(
+            spec["name"],
+            cases=cases,
+            runners=runners,
+            intervention_type=spec["intervention"]["type"],
+            intervention_version=spec["intervention"]["version"],
+            scorer=spec["scorer"],
+            gate_version=spec["gate_version"],
+            gates=ReplayGates(**spec["gates"]),
+            budget=ExperimentBudget(**spec["budget"]),
+            permission_ref=spec["permission_ref"],
+            frozen_at=spec["frozen_at"],
+            synthetic=spec["synthetic"],
+        )
+        if plan.experiment_id != saved["experiment_id"] or plan.to_dict() != spec:
+            raise ValueError("saved experiment differs from supplied frozen bindings")
+        return cls(plan, cases=cases, runners=runners, root=root)
+
+    @property
+    def experiment_id(self):
+        return self._plan.experiment_id
+
+    def _slots(self):
+        for case in self._spec["cases"]:
+            for runner in self._spec["runners"]:
+                yield (
+                    case,
+                    runner,
+                    slot_id(self.experiment_id, case["case_id"], runner["candidate_id"]),
+                )
+
+    def _ledger(self):
+        from agentloop.experiment_reports import read_experiment
+
+        if read_json(local_path(self.root, "experiment.json")) != {
+            "experiment_id": self.experiment_id,
+            "specification": self._spec,
+        }:
+            raise ValueError("frozen experiment declaration changed")
+        evidence = read_experiment(self.root)
+        if any(row["status"] == "unresolved" for row in evidence["rows"]):
+            raise ValueError("uncertain experiment attempt requires host reconciliation")
+        started = [row["admission"] for row in evidence["rows"] if row["admission"] is not None]
+        receipts = [row["receipt"] for row in evidence["rows"] if row["receipt"] is not None]
+        return started, receipts
+
+    def run(self, *, enabled=False):
+        if type(enabled) is not bool:
+            raise ValueError("experiment execution requires explicit boolean opt-in")
+        with journal_lock(self.root):
+            if not enabled:
+                return {"experiment_id": self.experiment_id, "enabled": False, "executed": 0}
+            if timestamp(utc_now_iso()) < timestamp(self._spec["frozen_at"]):
+                raise ValueError("experiment clock precedes its frozen declaration")
+            started, receipts = self._ledger()
+            finished = {receipt["slot_id"] for receipt in receipts}
+            executed, stop_reason = 0, None
+            budget = self._spec["budget"]
+            consumed_tokens = sum(item["runner"]["reservation"]["tokens"] or 0 for item in started)
+            consumed_cost = sum(
+                (Fraction(str(item["runner"]["reservation"]["cost_usd"] or 0)) for item in started),
+                Fraction(0),
+            )
+            halted = any(receipt["halted"] for receipt in receipts)
+            for declared_case, declared_runner, identity in self._slots():
+                if identity in finished:
+                    continue
+                if self._runners[declared_runner["candidate_id"]].declaration() != declared_runner:
+                    raise ValueError("runner declaration changed before admission")
+                if halted:
+                    stop_reason = "prior_budget_control"
+                    break
+                reservation = declared_runner["reservation"]
+                if (
+                    len(started) >= budget["max_invocations"]
+                    or (
+                        budget["max_tokens"] is not None
+                        and consumed_tokens + reservation["tokens"] > budget["max_tokens"]
+                    )
+                    or (
+                        budget["max_cost_usd"] is not None
+                        and consumed_cost + Fraction(str(reservation["cost_usd"]))
+                        > Fraction(str(budget["max_cost_usd"]))
+                    )
+                ):
+                    stop_reason = "execution_budget_exhausted"
+                    break
+                admission = {
+                    "experiment_id": self.experiment_id,
+                    "slot_id": identity,
+                    "case_id": declared_case["case_id"],
+                    "candidate_id": declared_runner["candidate_id"],
+                    "runner": declared_runner,
+                    "started_at": utc_now_iso(),
+                }
+                write_once(self.root, f"attempts/{identity}/started.json", admission)
+                remaining = BudgetLimits(
+                    max_tool_calls=budget["max_invocations"] - len(started),
+                    max_tokens=None
+                    if budget["max_tokens"] is None
+                    else budget["max_tokens"] - consumed_tokens,
+                    max_cost_usd=None
+                    if budget["max_cost_usd"] is None
+                    else float(Fraction(str(budget["max_cost_usd"])) - consumed_cost),
+                )
+                receipt, interruption = self._execute(
+                    declared_case, declared_runner, admission, remaining
+                )
+                started.append(admission)
+                receipts.append(receipt)
+                consumed_tokens += reservation["tokens"] or 0
+                consumed_cost += Fraction(str(reservation["cost_usd"] or 0))
+                halted = receipt["halted"]
+                executed += 1
+                if interruption is not None:
+                    raise interruption
+            return {
+                "experiment_id": self.experiment_id,
+                "enabled": True,
+                "executed": executed,
+                "completed_slots": len(receipts),
+                "stop_reason": stop_reason,
+            }
+
+    def _execute(self, declaration, runner_declaration, admission, remaining):
+        case, runner = (
+            self._cases[declaration["case_id"]],
+            self._runners[runner_declaration["candidate_id"]],
+        )
+        data = case.payload()
+        identity = admission["slot_id"]
+        started = utc_now_iso()
+        trace = AgentTrace(
+            "Experiment candidate: " + runner.candidate_id,
+            run_id="experiment_run_" + identity,
+            started_at=started,
+            metadata=workflow_metadata(
+                WorkflowInfo(
+                    self._spec["intervention"]["type"], self._spec["intervention"]["version"]
+                ),
+                task_id=case.example_id,
+                example_id=case.example_id,
+                status="running",
+                metadata={
+                    "synthetic": self._spec["synthetic"],
+                    "experiment_id": self.experiment_id,
+                    "case_id": case.case_id,
+                    "repetition": case.repetition,
+                    "condition": runner.candidate_id,
+                },
+            ),
+        )
+        run = Harness(
+            HarnessConfig("enforce", (budget_policy(remaining, metered_boundaries=("tool",)),))
+        ).start_run(trace.run_id)
+        request = ExperimentRequest(
+            self.experiment_id,
+            case.case_id,
+            runner.candidate_id,
+            data["input_ref"],
+            monotonic() + self._spec["budget"]["timeout_s"],
+            canonical(data["inputs"]),
+        )
+        invoked, result, error, interruption = False, None, None, None
+        status = "completed"
+        root_event_id = "experiment_root_" + identity
+
+        def invoke():
+            nonlocal invoked, result
+            if request.remaining_s <= 0:
+                raise TimeoutError("candidate admission deadline expired")
+            invoked = True
+            with bind_trace_context(trace, root_event_id):
+                value = runner.invoke(request)
+            if type(value) is not ExperimentResult:
+                if inspect.iscoroutine(value) or inspect.isgenerator(value):
+                    value.close()
+                raise ValueError("runner returned an invalid result")
+            result = value
+            return value
+
+        began = perf_counter()
+        with bind_trace_context(trace):
+            try:
+                run.wrap(
+                    invoke,
+                    boundary="tool",
+                    branch_id="experiment",
+                    dispatch=DispatchOptions(reservation=runner.reservation),
+                    usage_reader=lambda value: value.usage,
+                )()
+                if runner.declaration() != runner_declaration:
+                    raise ValueError("runner declaration changed")
+            except HarnessControlError:
+                status, error = "stopped", "budget_control"
+            except TimeoutError:
+                status, error = "timed_out", "runner_timeout"
+            except Exception:
+                status, error = "failed", "runner_error"
+            except BaseException as exc:
+                status, error, interruption = "cancelled", "runner_cancelled", exc
+        elapsed = (perf_counter() - began) * 1000
+        if status == "completed" and request.remaining_s <= 0:
+            status, error = "timed_out", "runner_timeout"
+        ended = utc_now_iso()
+        trace.ended_at, trace.elapsed_ms = ended, elapsed
+        workflow_status = (
+            "completed"
+            if status == "completed"
+            else "cancelled"
+            if status == "cancelled"
+            else "failed"
+        )
+        trace.metadata["agentloop.workflow"]["status"] = workflow_status
+        trace.metadata[EXPERIMENT_KEY] = {
+            "schema_version": "1.0",
+            "experiment_id": self.experiment_id,
+            "slot_id": identity,
+            "runner": runner_declaration,
+            "status": status,
+            "invoked": invoked,
+        }
+        if invoked:
+            record_operation(
+                "candidate runner",
+                kind="workflow",
+                trace=trace,
+                event_id=root_event_id,
+                started_at=started,
+                ended_at=ended,
+                duration_ms=elapsed,
+                status="ok" if status == "completed" else "error",
+                metadata={"error_type": error} if error else None,
+            )
+        try:
+            baseline = AgentTrace.from_dict(data["baseline"])
+            fixture = {
+                "schema_version": "2.0",
+                "id": case.case_id,
+                "input_ref": data["input_ref"],
+                "expected_ref": data["quality_ref"],
+                "expected": data["expected"],
+                "scorer": self._spec["scorer"],
+                "baseline_output": data["baseline_output"],
+                "candidate_status": "completed"
+                if status == "completed"
+                else "cancelled"
+                if status == "cancelled"
+                else "timed_out"
+                if status == "timed_out"
+                else "failed",
+            }
+            if status == "completed":
+                fixture["candidate_output"] = result.output
+            quality = build_quality_report(
+                [fixture],
+                baseline_trace=baseline,
+                candidate_trace=trace,
+                min_score=self._spec["gates"]["min_quality_score"],
+            )
+            trace.metadata.update(
+                success=status == "completed" and quality["passed"],
+                quality_score=quality["candidate_score"],
+            )
+            record = build_intervention(
+                baseline,
+                trace,
+                target_finding_ids=declaration["source_finding_ids"],
+                intervention_type=self._spec["intervention"]["type"],
+                configuration={
+                    "experiment_id": self.experiment_id,
+                    "slot_id": identity,
+                    "intervention_version": self._spec["intervention"]["version"],
+                    "runner": runner_declaration,
+                    "gate_version": self._spec["gate_version"],
+                },
+                diagnosis=data["diagnosis"],
+                gates=ReplayGates(**self._spec["gates"]),
+                quality_report=quality,
+            )
+            documents = {
+                "trace.json": trace.to_dict(),
+                "quality.json": quality,
+                "intervention.json": record.to_dict(),
+            }
+            for filename, document in documents.items():
+                write_once(self.root, f"attempts/{identity}/{filename}", document)
+            receipt = {
+                "schema_version": "1.0",
+                "experiment_id": self.experiment_id,
+                "slot_id": identity,
+                "started_hash": fingerprint(admission),
+                "case_id": case.case_id,
+                "candidate_id": runner.candidate_id,
+                "status": status,
+                "error_category": error,
+                "invoked": invoked,
+                "halted": run.stopped,
+                "budget_call_id": run.results[0].call_id,
+                "budget_config_hash": run.harness.config.config_hash,
+                "budget_dispatched": run.results[-1].dispatched,
+                "usage": None if result is None else result.usage.to_dict(),
+                "output_hash": None if result is None else fingerprint(result.output),
+                "quality_passed": quality["passed"],
+                "gates_passed": record.to_dict()["gates_passed"],
+                "artifact_hashes": {name: fingerprint(value) for name, value in documents.items()},
+            }
+            write_once(self.root, f"attempts/{identity}/receipt.json", receipt)
+            return receipt, interruption
+        except BaseException:
+            if interruption is not None:
+                raise interruption
+            raise

agentloop/exporters.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from agentloop.costs import format_cost_usd
+from agentloop.markdown import (
+    markdown_code_span,
+    markdown_heading,
+    markdown_table_cell,
+    markdown_text,
+)
+from agentloop.timing import format_duration_ms
+
+
+def export_report_json(report: dict[str, Any], path: str | Path) -> Path:
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(report, indent=2), encoding="utf-8")
+    return out
+
+
+def export_report_markdown(report: dict[str, Any], path: str | Path) -> Path:
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    lines = [
+        f"# AgentLoop Report: {markdown_heading(report['name'])}",
+        "",
+        f"- Run ID: {markdown_code_span(report['run_id'])}",
+        f"- Runtime: {format_duration_ms(report['total_runtime_ms'])}",
+        "- Estimated cost: "
+        + format_cost_usd(report.get("estimated_cost_usd"), report.get("cost_status", "complete")),
+        f"- Model calls: {report['model_call_count']}",
+        f"- Tool calls: {report['tool_call_count']}",
+        f"- Retries: {report['retry_count']}",
+        f"- Input tokens: {report['input_tokens']}",
+        f"- Output tokens: {report['output_tokens']}",
+        "- Repeated context: "
+        + (
+            "unavailable"
+            if report["repeated_context_ratio"] is None
+            else f"{report['repeated_context_ratio']:.1%}"
+        ),
+        "",
+        "## Recommendations",
+    ]
+    for rec in report["recommendations"]:
+        lines.append(f"- **{markdown_text(rec['title'])}** — {markdown_text(rec['description'])}")
+    if "execution" in report:
+        lines.extend(
+            [
+                "",
+                "## Workflow",
+                "",
+                markdown_code_span(json.dumps(report["execution"], sort_keys=True)),
+                "",
+                "Cost and token totals cover recorded model calls, not arbitrary service billing.",
+            ]
+        )
+    lines.extend(
+        [
+            "",
+            "## Events",
+            "",
+            "| Type | Name | Duration ms | Model | Input | Output | Status |",
+            "|---|---|---:|---|---:|---:|---|",
+        ]
+    )
+    for event in report["events"]:
+        lines.append(
+            f"| {markdown_table_cell(event['event_type'])} | "
+            f"{markdown_table_cell(event['name'])} | "
+            + ("unavailable" if event["duration_ms"] is None else f"{event['duration_ms']:.2f}")
+            + " | "
+            f"{markdown_table_cell(event.get('model') or '')} | "
+            f"{event.get('input_tokens', 0)} | {event.get('output_tokens', 0)} | "
+            f"{markdown_table_cell(event.get('status', 'ok'))} |"
+        )
+    if report.get("stages"):
+        lines.extend(
+            [
+                "",
+                "## Stages",
+                "",
+                "| Span | Stage / version | Kind | Outcome | Schema references | Data references | Dependencies |",
+                "| --- | --- | --- | --- | --- | --- | --- |",
+            ]
+        )
+        for identity, stage in report["stages"].items():
+            cells = [
+                identity,
+                f"{stage.get('stage_id')} / {stage.get('version')}",
+                stage.get("kind"),
+                stage.get("outcome"),
+                f"{stage.get('input_schema_ref')} -> {stage.get('output_schema_ref')}",
+                f"{stage.get('input_ref')} -> {stage.get('output_ref')}",
+                ", ".join(stage.get("depends_on", [])),
+            ]
+            lines.append(
+                "| "
+                + " | ".join(
+                    markdown_table_cell("unavailable" if value is None else str(value))
+                    for value in cells
+                )
+                + " |"
+            )
+    if "semantic_judgments" in report:
+        from agentloop.judgment_views import judgment_markdown
+
+        lines.extend(judgment_markdown(report["semantic_judgments"]))
+    if "semantic_waste" in report:
+        from agentloop.semantic_waste import semantic_waste_markdown
+
+        lines.extend(semantic_waste_markdown(report["semantic_waste"]))
+    out.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return out

agentloop/finding_benchmark_types.py

@@ -0,0 +1,131 @@
+"""Frozen native fixtures and independent labels for finding evaluation."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+
+from agentloop.judgment_types import canonical, fingerprint, text
+from agentloop.tracer import AgentTrace
+
+FINDING_BENCHMARK_VERSION = "1.0"
+LABELS = {"opportunity", "no_opportunity", "unknown", "ambiguous"}
+
+
+def _keys(value, expected, name):
+    if not isinstance(value, dict) or set(value) != set(expected.split()):
+        raise ValueError(f"invalid {name} fields")
+
+
+@dataclass(frozen=True, init=False)
+class FindingBenchmark:
+    """Own a validated JSON snapshot; returned dictionaries cannot mutate the plan.
+
+    Labels enumerate justified findings by exact family and affected-span set.
+    They are never passed to detectors. Group declarations prevent splitting one
+    workload family across development and evaluation, not all possible leakage.
+    """
+
+    _json: str
+
+    def __init__(self, value):
+        owned = json.loads(canonical(value))
+        _keys(owned, "schema_version name version provenance_ref synthetic policy cases", "corpus")
+        if owned["schema_version"] != FINDING_BENCHMARK_VERSION:
+            raise ValueError("unsupported finding benchmark version")
+        for key in ("name", "version", "provenance_ref"):
+            text(owned[key], key)
+        if type(owned["synthetic"]) is not bool:
+            raise ValueError("synthetic must be boolean")
+        policy = owned["policy"]
+        _keys(policy, "id version ref", "frozen policy")
+        for key, val in policy.items():
+            text(val, key)
+        cases = owned["cases"]
+        if not isinstance(cases, list) or not 1 <= len(cases) <= 10000:
+            raise ValueError("corpus requires 1..10000 cases")
+        seen, groups, traces = set(), {}, {}
+        span_count = 0
+        for case in cases:
+            _keys(
+                case,
+                "id workload_ref group split scenario rule_id label label_ref label_version "
+                "rationale opportunities required_evidence trace",
+                "case",
+            )
+            for key in (
+                "id",
+                "workload_ref",
+                "group",
+                "scenario",
+                "rule_id",
+                "label_ref",
+                "label_version",
+                "rationale",
+            ):
+                text(case[key], key)
+            if case["id"] in seen:
+                raise ValueError("case IDs must be unique")
+            seen.add(case["id"])
+            split = case["split"]
+            if split not in {"development", "evaluation"} or case["label"] not in LABELS:
+                raise ValueError("invalid split or label")
+            if groups.setdefault(case["group"], split) != split:
+                raise ValueError("workload groups must not cross splits")
+            trace = AgentTrace.from_dict(case["trace"])
+            span_count += len(trace.events)
+            if span_count > 100000:
+                raise ValueError("corpus exceeds 100000 total spans")
+            ids = {event.event_id for event in trace.events}
+            if len(ids) != len(trace.events) or len(ids) > 10000:
+                raise ValueError("trace requires unique span IDs and at most 10000 spans")
+            if traces.setdefault(fingerprint(case["trace"]), split) != split:
+                raise ValueError("duplicate traces must not cross splits")
+            opportunities = case["opportunities"]
+            if not isinstance(opportunities, list):
+                raise ValueError("opportunities must be a list")
+            signatures = set()
+            for opportunity in opportunities:
+                _keys(opportunity, "family spans", "opportunity")
+                text(opportunity["family"], "family")
+                spans = opportunity["spans"]
+                if (
+                    not isinstance(spans, list)
+                    or not spans
+                    or any(not isinstance(span, str) or span not in ids for span in spans)
+                    or len(set(spans)) != len(spans)
+                ):
+                    raise ValueError("opportunities require unique existing spans")
+                signature = (opportunity["family"], tuple(sorted(spans)))
+                if signature in signatures:
+                    raise ValueError("duplicate opportunity")
+                signatures.add(signature)
+            if bool(opportunities) != (case["label"] == "opportunity"):
+                raise ValueError("only opportunity labels enumerate justified findings")
+            evidence = case["required_evidence"]
+            if (
+                not isinstance(evidence, list)
+                or any(
+                    not isinstance(item, str)
+                    or item not in {"timing", "tokens", "cost", "semantic"}
+                    for item in evidence
+                )
+                or len(set(evidence)) != len(evidence)
+            ):
+                raise ValueError("unsupported evidence requirement")
+        object.__setattr__(self, "_json", canonical(owned))
+
+    def to_dict(self):
+        return json.loads(self._json)
+
+    @property
+    def corpus_hash(self):
+        return fingerprint(self.to_dict())
+
+    def cases(self, split):
+        if split not in {"development", "evaluation"}:
+            raise ValueError("select development or evaluation explicitly")
+        cases = [item for item in self.to_dict()["cases"] if item["split"] == split]
+        if not cases:
+            raise ValueError("selected split is empty")
+        return cases