agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/interventions.py

@@ -0,0 +1,190 @@
+"""Immutable evidence linking finding predictions to an observed replay."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from agentloop.findings import build_diagnosis
+from agentloop.harness_evidence import HarnessEvidenceError, comparison_evidence
+from agentloop.replay import ReplayGates, build_replay_report
+
+INTERVENTION_SCHEMA_VERSION = "1.0"
+_IDENTITY_FIELDS = (
+    "schema_version",
+    "baseline_run_id",
+    "candidate_run_id",
+    "target_finding_ids",
+    "intervention_type",
+    "configuration",
+)
+
+
+class InterventionValidationError(ValueError):
+    """An intervention does not satisfy the evidence contract."""
+
+
+class InterventionReferenceError(LookupError):
+    """A required trace or baseline finding is not available in this project."""
+
+
+class InterventionConflictError(ValueError):
+    """The same intervention identity already has different evidence."""
+
+
+def canonical_json(value: Any) -> str:
+    try:
+        return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
+    except (TypeError, ValueError) as exc:
+        raise InterventionValidationError("evidence must contain finite JSON values") from exc
+
+
+def trace_fingerprint(trace: Any) -> str:
+    return hashlib.sha256(canonical_json(trace.to_dict()).encode("utf-8")).hexdigest()
+
+
+def _identity(payload: dict[str, Any]) -> str:
+    identity = {key: payload[key] for key in _IDENTITY_FIELDS}
+    return "int_" + hashlib.sha256(canonical_json(identity).encode("utf-8")).hexdigest()
+
+
+@dataclass(frozen=True)
+class InterventionRecord:
+    """Canonical JSON owns the snapshot; callers receive independent copies."""
+
+    _json: str
+
+    @property
+    def intervention_id(self) -> str:
+        return self.to_dict()["intervention_id"]
+
+    def to_dict(self) -> dict[str, Any]:
+        return json.loads(self._json)
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> InterventionRecord:
+        if (
+            not isinstance(payload, dict)
+            or payload.get("schema_version") != INTERVENTION_SCHEMA_VERSION
+        ):
+            raise InterventionValidationError("unsupported intervention schema_version")
+        for key in ("baseline_run_id", "candidate_run_id", "intervention_type"):
+            if not isinstance(payload.get(key), str) or not payload[key].strip():
+                raise InterventionValidationError(f"{key} must be a nonempty string")
+        if payload["baseline_run_id"] == payload["candidate_run_id"]:
+            raise InterventionValidationError("baseline and candidate must have distinct run IDs")
+        targets = payload.get("target_finding_ids")
+        if (
+            not isinstance(targets, list)
+            or not targets
+            or any(not isinstance(item, str) or not item for item in targets)
+        ):
+            raise InterventionValidationError("target_finding_ids must be a nonempty list of IDs")
+        if targets != sorted(set(targets)):
+            raise InterventionValidationError("target_finding_ids must be sorted and unique")
+        for key in ("configuration", "metadata", "predicted", "measured", "trace_fingerprints"):
+            if not isinstance(payload.get(key), dict):
+                raise InterventionValidationError(f"{key} must be an object")
+        findings = payload["predicted"].get("findings")
+        if not isinstance(findings, list) or any(not isinstance(item, dict) for item in findings):
+            raise InterventionValidationError("predicted.findings must contain finding snapshots")
+        if [item.get("finding_id") for item in findings] != targets:
+            raise InterventionValidationError("prediction snapshots must match target finding IDs")
+        measured = payload["measured"]
+        for condition in ("baseline", "candidate"):
+            if (
+                not isinstance(measured.get(condition), dict)
+                or measured[condition].get("run_id") != payload[f"{condition}_run_id"]
+            ):
+                raise InterventionValidationError("replay run IDs must match the intervention")
+            fingerprint = payload["trace_fingerprints"].get(condition)
+            if (
+                not isinstance(fingerprint, str)
+                or len(fingerprint) != 64
+                or any(char not in "0123456789abcdef" for char in fingerprint)
+            ):
+                raise InterventionValidationError("trace fingerprints must be SHA-256 hex digests")
+        if not isinstance(measured.get("deltas"), dict) or not isinstance(
+            measured.get("gates"), dict
+        ):
+            raise InterventionValidationError("measured must retain replay deltas and gates")
+        if (
+            type(payload.get("gates_passed")) is not bool
+            or measured["gates"].get("passed") is not payload["gates_passed"]
+        ):
+            raise InterventionValidationError("gates_passed must match the replay decision")
+        if payload.get("intervention_id") != _identity(payload):
+            raise InterventionValidationError("intervention_id does not match its identity fields")
+        return cls(canonical_json(payload))
+
+
+def build_intervention(
+    baseline: Any,
+    candidate: Any,
+    *,
+    target_finding_ids: list[str],
+    intervention_type: str,
+    configuration: dict[str, Any] | None = None,
+    metadata: dict[str, Any] | None = None,
+    diagnosis: dict[str, Any] | None = None,
+    gates: ReplayGates | None = None,
+    quality_report: dict[str, Any] | None = None,
+    replay_report: dict[str, Any] | None = None,
+) -> InterventionRecord:
+    """Capture prediction snapshots, then attach the observed pair comparison."""
+    if (
+        not isinstance(target_finding_ids, list)
+        or not target_finding_ids
+        or any(not isinstance(item, str) or not item for item in target_finding_ids)
+    ):
+        raise InterventionValidationError("at least one target finding ID is required")
+    diagnosis = build_diagnosis(baseline) if diagnosis is None else diagnosis
+    if diagnosis.get("run_id") != baseline.run_id:
+        raise InterventionValidationError("diagnosis must belong to the baseline run")
+    targets = sorted(set(target_finding_ids))
+    available = {finding["finding_id"]: finding for finding in diagnosis.get("findings", [])}
+    if any(target not in available for target in targets):
+        raise InterventionReferenceError("target baseline finding not found")
+    replay = (
+        replay_report
+        if replay_report is not None
+        else build_replay_report(
+            baseline,
+            candidate,
+            gates=gates,
+            quality_report=quality_report,
+        )
+    )
+    if metadata is not None and not isinstance(metadata, dict):
+        raise InterventionValidationError("metadata must be an object")
+    record_metadata = dict(metadata or {})
+    if "agentloop.harness_evidence" in record_metadata:
+        raise InterventionValidationError(
+            "agentloop.harness_evidence is derived from the compared traces"
+        )
+    try:
+        harness_evidence = comparison_evidence(baseline, candidate)
+    except HarnessEvidenceError:
+        raise InterventionValidationError("invalid harness evidence in compared traces") from None
+    if harness_evidence is not None:
+        record_metadata["agentloop.harness_evidence"] = harness_evidence
+    payload = {
+        "schema_version": INTERVENTION_SCHEMA_VERSION,
+        "baseline_run_id": baseline.run_id,
+        "candidate_run_id": candidate.run_id,
+        "target_finding_ids": targets,
+        "intervention_type": intervention_type,
+        "configuration": {} if configuration is None else configuration,
+        "predicted": {"findings": [available[target] for target in targets]},
+        "measured": replay,
+        "gates_passed": replay["gates"]["passed"],
+        "trace_fingerprints": {
+            "baseline": trace_fingerprint(baseline),
+            "candidate": trace_fingerprint(candidate),
+        },
+        "metadata": record_metadata,
+    }
+    payload["intervention_id"] = _identity(payload)
+    return InterventionRecord.from_dict(payload)

agentloop/issues.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+from typing import Any
+
+from agentloop.costs import format_cost_usd
+from agentloop.markdown import markdown_code_span, markdown_heading, markdown_text
+from agentloop.timing import format_duration_ms
+
+# Generated drafts target this repository, so use only deliberately maintained
+# labels that already exist. Finding type and severity remain structured fields
+# in the draft body instead of creating an unbounded dynamic label taxonomy.
+ISSUE_DRAFT_LABELS = ("enhancement",)
+
+
+def build_issue_drafts(queue: list[dict[str, Any]], *, limit: int = 5) -> list[dict[str, Any]]:
+    drafts = []
+    for item in queue:
+        if item.get("patchable_count", 0) <= 0:
+            continue
+        drafts.append(_issue_for_queue_item(item))
+        if len(drafts) >= limit:
+            break
+    return drafts
+
+
+def issue_drafts_to_markdown(drafts: list[dict[str, Any]]) -> str:
+    lines = ["# AgentLoop GitHub Issue Drafts", ""]
+    if not drafts:
+        lines.append("No patchable optimization queue items found.")
+        return "\n".join(lines).rstrip() + "\n"
+
+    for draft in drafts:
+        lines.extend(
+            [
+                f"## {markdown_heading(draft['title'])}",
+                "",
+                f"- Labels: {markdown_text(', '.join(map(str, draft['labels'])))}",
+                "",
+                draft["body"],
+                "",
+            ]
+        )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _issue_for_queue_item(item: dict[str, Any]) -> dict[str, Any]:
+    title = f"Optimize agent workflow: {item['title']}"
+    labels = list(ISSUE_DRAFT_LABELS)
+    body = "\n".join(
+        [
+            "AgentLoop detected a recurring patchable optimization opportunity.",
+            "",
+            "## Finding",
+            "",
+            f"- Type: {markdown_code_span(item['type'])}",
+            f"- Severity: {markdown_code_span(item['severity'])}",
+            f"- Occurrences: {item['occurrence_count']}",
+            f"- Affected runs: {item['run_count']}",
+            f"- Patchable findings: {item['patchable_count']}",
+            f"- Quality risk: {markdown_code_span(item.get('quality_risk', 'unknown'))}",
+            f"- Requires scorer: {item.get('requires_scorer', True)}",
+            f"- Safe to auto-patch: {item.get('safe_to_auto_patch', False)}",
+            f"- Estimated latency savings: {format_duration_ms(item['estimated_latency_savings_ms'])}",
+            "- Estimated cost savings: " + format_cost_usd(item.get("estimated_cost_savings_usd")),
+            f"- Priority score: {item['priority_score']:.1f}",
+            "",
+            "## Affected Runs",
+            "",
+            "\n".join(
+                f"- {markdown_code_span(run_id)}" for run_id in item.get("affected_runs", [])[:20]
+            ),
+            "",
+            "## Acceptance Criteria",
+            "",
+            "- Generate or inspect the AgentLoop patch plan for one affected run.",
+            "- Apply a constrained workflow rewrite.",
+            "- Run `agentloop replay` against baseline and candidate traces.",
+            "- Attach a quality fixture or scorer when quality risk is medium or high.",
+            "- Run `agentloop ci` and confirm cost, latency, retry, schema, and quality gates pass.",
+        ]
+    )
+    return {
+        "title": title,
+        "body": body,
+        "labels": labels,
+        "queue_id": item["queue_id"],
+        "type": item["type"],
+        "severity": item["severity"],
+        "priority_score": item["priority_score"],
+    }

agentloop/judgment_adapters.py

@@ -0,0 +1,125 @@
+"""Replaceable local prediction and typed-service bridges for judgment contract 1.0."""
+
+from __future__ import annotations
+
+import inspect
+import json
+from dataclasses import dataclass
+from typing import Callable
+
+from agentloop.judgment_types import (
+    JudgeIdentity,
+    JudgeUsage,
+    JudgmentAnswer,
+    JudgmentUncertainty,
+    canonical,
+)
+
+
+def _callback(value, name):
+    if (
+        not callable(value)
+        or inspect.iscoroutinefunction(value)
+        or inspect.isasyncgenfunction(value)
+    ):
+        raise ValueError(f"{name} must be a synchronous callable")
+
+
+@dataclass(frozen=True)
+class LocalPredictorJudge:
+    """Adapt a local scalar predictor without assuming its model framework or labels.
+
+    The predictor accepts a JudgmentRequest and keyword timeout_s. The session
+    validates its output against the request's type/domain. Usage is caller-owned
+    metadata; no zero token/cost claim is inferred from the word 'local'.
+    """
+
+    identity: JudgeIdentity
+    predict: Callable
+    usage: JudgeUsage = JudgeUsage()
+
+    def __post_init__(self):
+        if type(self.identity) is not JudgeIdentity or type(self.usage) is not JudgeUsage:
+            raise ValueError("predictor requires a typed identity and usage")
+        _callback(self.predict, "predict")
+
+    def judge(self, request, *, timeout_s=None):
+        value = self.predict(request, timeout_s=timeout_s)
+        if type(value) is JudgmentAnswer:
+            return value
+        if inspect.iscoroutine(value) or inspect.isgenerator(value):
+            value.close()
+            raise ValueError("predictors must return a scalar, not deferred work")
+        return JudgmentAnswer(value, usage=self.usage)
+
+
+def parse_typed_answer(response):
+    """Accept a finite structured response, never free-form prose or extra fields."""
+    if not isinstance(response, dict) or set(response) - {
+        "value",
+        "status",
+        "reason",
+        "usage",
+        "uncertainty",
+    }:
+        raise ValueError("service response must be a typed answer object")
+    encoded = canonical(response)
+    if len(encoded.encode("utf-8")) > 65536:
+        raise ValueError("service answer exceeds the 64 KiB contract limit")
+    owned = json.loads(encoded)
+    return JudgmentAnswer(
+        value=owned.get("value"),
+        status=owned.get("status", "known"),
+        reason=owned.get("reason"),
+        usage=JudgeUsage(**owned.get("usage", {})),
+        uncertainty=JudgmentUncertainty(**owned.get("uncertainty", {})),
+    )
+
+
+@dataclass(frozen=True)
+class TypedServiceJudge:
+    """Adapt an optional host-owned transport using AgentLoop's structured schema.
+
+    Credentials and client SDKs belong to the host transport. This adapter never
+    reads environment secrets, imports a provider, or opens a network connection
+    itself. A configured transport may perform remote work only during explicit
+    session execution. The transport owns connection/response byte limits and
+    cooperative cancellation before returning a decoded JSON object.
+    """
+
+    identity: JudgeIdentity
+    transport: Callable | None = None
+    credentials_required: bool = True
+    credentials_available: bool = False
+
+    def __post_init__(self):
+        if type(self.identity) is not JudgeIdentity:
+            raise ValueError("service requires JudgeIdentity")
+        if (
+            type(self.credentials_required) is not bool
+            or type(self.credentials_available) is not bool
+        ):
+            raise ValueError("credential availability must be a boolean declaration")
+        if self.transport is not None:
+            _callback(self.transport, "transport")
+
+    @property
+    def availability(self):
+        if self.transport is None:
+            return "not_configured"
+        if self.credentials_required and not self.credentials_available:
+            return "missing_credentials"
+        return "ready"
+
+    def judge(self, request, *, timeout_s=None):
+        if self.availability != "ready":
+            return JudgmentAnswer(
+                status="unknown",
+                reason="abstained",
+                usage=JudgeUsage(0, 0, 0, "not_dispatched", "not_dispatched"),
+            )
+        response = self.transport(request.to_dict(), timeout_s=timeout_s)
+        if inspect.iscoroutine(response) or inspect.isgenerator(response):
+            response.close()
+            raise ValueError("transport must return a decoded JSON answer")
+        return parse_typed_answer(response)

agentloop/judgment_benchmark_types.py

@@ -0,0 +1,98 @@
+"""Frozen examples and execution declarations for backend comparisons."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from typing import Any
+
+from agentloop.judgment_types import JudgmentRequest, canonical, fingerprint, finite, text
+
+BENCHMARK_VERSION = "1.0"
+
+
+@dataclass(frozen=True)
+class JudgmentExample:
+    example_id: str
+    request: JudgmentRequest
+    expected: Any = None
+    label_status: str = "unlabelled"
+    label_ref: str | None = None
+    label_version: str | None = None
+
+    def __post_init__(self):
+        text(self.example_id, "example_id")
+        if type(self.request) is not JudgmentRequest:
+            raise ValueError("example requires JudgmentRequest")
+        if self.label_status not in {"independent", "unlabelled"}:
+            raise ValueError("labels must be independently sourced or unavailable")
+        if self.label_status == "independent":
+            text(self.label_ref, "label_ref")
+            text(self.label_version, "label_version")
+            valid = (
+                type(self.expected) is bool
+                if self.request.spec.kind == "probability"
+                else self.request.spec.accepts(self.expected)
+            )
+            if not valid:
+                raise ValueError("reference label violates the example's type/domain")
+        elif (
+            self.expected is not None
+            or self.label_ref is not None
+            or self.label_version is not None
+        ):
+            raise ValueError("unlabelled examples cannot carry an expected answer")
+
+    def to_dict(self):
+        return json.loads(canonical(asdict(self)))
+
+    @classmethod
+    def from_dict(cls, value):
+        owned = dict(value)
+        owned["request"] = JudgmentRequest.from_dict(owned["request"])
+        return cls(**owned)
+
+
+@dataclass(frozen=True)
+class JudgmentBenchmark:
+    name: str
+    version: str
+    examples: tuple[JudgmentExample, ...]
+    repetitions: int = 1
+    seed: int = 0
+    timeout_s: float | None = None
+    synthetic: bool = False
+
+    def __post_init__(self):
+        text(self.name, "name")
+        text(self.version, "version")
+        if (
+            not isinstance(self.examples, (list, tuple))
+            or not self.examples
+            or any(type(item) is not JudgmentExample for item in self.examples)
+        ):
+            raise ValueError("benchmark requires a nonempty sequence of JudgmentExample")
+        object.__setattr__(self, "examples", tuple(self.examples))
+        if len({item.example_id for item in self.examples}) != len(self.examples):
+            raise ValueError("example IDs must be unique")
+        if type(self.repetitions) is not int or self.repetitions < 1:
+            raise ValueError("repetitions must be a positive integer")
+        if type(self.seed) is not int or type(self.synthetic) is not bool:
+            raise ValueError("seed and synthetic require an integer and boolean")
+        if self.timeout_s is not None and (not finite(self.timeout_s) or self.timeout_s <= 0):
+            raise ValueError("timeout_s must be finite and positive")
+
+    def to_dict(self):
+        return {"schema_version": BENCHMARK_VERSION, **json.loads(canonical(asdict(self)))}
+
+    @property
+    def protocol_hash(self):
+        return fingerprint(self.to_dict())
+
+    @classmethod
+    def from_dict(cls, value):
+        owned = dict(value)
+        if owned.pop("schema_version") != BENCHMARK_VERSION:
+            raise ValueError("unsupported judgment benchmark version")
+        owned["examples"] = tuple(JudgmentExample.from_dict(item) for item in owned["examples"])
+        return cls(**owned)