agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/findings.py

@@ -0,0 +1,419 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import Any
+
+from agentloop.costs import format_cost_usd
+from agentloop.estimates import estimate_markdown
+from agentloop.markdown import markdown_code_span, markdown_heading, markdown_text
+from agentloop.optimizer import build_optimization_plan
+from agentloop.ranking import RANKING_KEY, RANKING_VERSION, rank_findings
+from agentloop.timing import format_duration_ms
+
+
+@dataclass
+class OptimizationFinding:
+    finding_id: str
+    severity: str
+    type: str
+    title: str
+    confidence: str
+    affected_spans: list[str]
+    evidence: list[dict[str, Any]]
+    savings: dict[str, Any]
+    rewrite: dict[str, Any]
+    validation: dict[str, Any]
+    metadata: dict[str, Any] = field(default_factory=dict)
+    evidence_level: str | None = None
+    assumptions: list[str] = field(default_factory=list)
+    observations: dict[str, Any] = field(default_factory=dict)
+    rule_id: str | None = None
+    rule_version: str | None = None
+    estimate: dict[str, Any] | None = None
+    ranking_requirements: tuple[str, ...] = ()
+
+    def to_dict(self) -> dict[str, Any]:
+        result = {
+            "finding_id": self.finding_id,
+            "severity": self.severity,
+            "type": self.type,
+            "title": self.title,
+            "confidence": self.confidence,
+            "affected_spans": self.affected_spans,
+            "evidence": self.evidence,
+            "savings": self.savings,
+            "rewrite": self.rewrite,
+            "validation": self.validation,
+            "metadata": self.metadata,
+        }
+        if self.evidence_level is not None:
+            result.update(
+                evidence_level=self.evidence_level,
+                assumptions=self.assumptions,
+                observations=self.observations,
+            )
+        if self.rule_id is not None:
+            result.update(rule_id=self.rule_id, rule_version=self.rule_version)
+        if self.estimate is not None:
+            result["estimate"] = deepcopy(self.estimate)
+        if self.ranking_requirements:
+            result["ranking_requirements"] = list(self.ranking_requirements)
+        return result
+
+
+def build_diagnosis(trace: Any, *, sort_by="priority", ranking_inputs=None) -> dict[str, Any]:
+    plan = build_optimization_plan(trace)
+    node_lookup = {node["node_id"]: node for node in plan.get("graph", {}).get("nodes", [])}
+    findings = [
+        _finding_from_card(card, plan, node_lookup=node_lookup)
+        for card in plan.get("optimization_cards", [])
+    ]
+    findings = [finding for finding in findings if finding is not None]
+    annotations = trace.metadata.get(RANKING_KEY, {}) if ranking_inputs is None else ranking_inputs
+    ranked = rank_findings(
+        [finding.to_dict() for finding in findings], inputs=annotations, sort_by=sort_by
+    )
+    return {
+        **({"semantic_waste": plan["semantic_waste"]} if "semantic_waste" in plan else {}),
+        **(
+            {
+                "semantic_judgments": plan["semantic_judgments"],
+                "evidence_categories": plan["evidence_categories"],
+            }
+            if "semantic_judgments" in plan
+            else {}
+        ),
+        **({"execution": plan["graph"]["execution"]} if "execution" in plan["graph"] else {}),
+        "run_id": plan["run_id"],
+        "name": plan["name"],
+        "cost_status": plan.get("cost_status", "complete"),
+        "current": plan["current"],
+        "estimated_after": plan["estimated_after"],
+        "summary": _summary(plan, findings),
+        "findings": ranked,
+        "ranking": {
+            "schema_version": RANKING_VERSION,
+            "sort_by": sort_by,
+            "ready_to_test_count": sum(
+                finding["ranking"]["status"] == "ready_to_test" for finding in ranked
+            ),
+            "interpretation": "Investigation ordering only. Unknown inputs remain explicit; estimated savings are not measured outcomes.",
+        },
+        "graph": plan["graph"],
+        "rule_errors": plan.get("rule_errors", []),
+        "analysis_complete": plan.get("analysis_complete", True),
+    }
+
+
+def diagnosis_to_markdown(diagnosis: dict[str, Any]) -> str:
+    current = diagnosis["current"]
+    after = diagnosis["estimated_after"]
+    lines = [
+        f"# AgentLoop Diagnosis: {markdown_heading(diagnosis['name'])}",
+        "",
+        f"- Run ID: {markdown_code_span(diagnosis['run_id'])}",
+        f"- Current runtime: {format_duration_ms(current['runtime_ms'])}",
+        f"- Estimated runtime after fixes: {format_duration_ms(after['runtime_ms'])}",
+        "- Estimated latency reduction: "
+        + (
+            "unavailable"
+            if after["latency_reduction_pct"] is None
+            else f"{after['latency_reduction_pct']:.2f}%"
+        ),
+        "- Current cost: "
+        + format_cost_usd(current.get("estimated_cost_usd"), current.get("cost_status")),
+        "- Estimated cost after fixes: "
+        + format_cost_usd(after.get("estimated_cost_usd"), after.get("cost_status")),
+        "- Estimated cost reduction: "
+        + (
+            "unavailable"
+            if after.get("cost_reduction_pct") is None
+            else f"{after['cost_reduction_pct']:.2f}%"
+        ),
+        "",
+        "## Findings",
+        "",
+    ]
+    findings = diagnosis.get("findings", [])
+    summary = diagnosis.get("summary")
+    if isinstance(summary, dict) and (
+        summary.get("unmodeled_latency_findings") or summary.get("unmodeled_cost_findings")
+    ):
+        lines.extend(["Some savings are unavailable; totals cover modeled candidates only.", ""])
+    if diagnosis.get("rule_errors"):
+        lines.extend(["Analysis incomplete; some finding rules failed:", ""])
+        for error in diagnosis["rule_errors"]:
+            lines.append(
+                f"- {markdown_code_span(error['rule_id'])}: {markdown_text(error['message'])}"
+            )
+        lines.append("")
+    if not findings:
+        lines.append("No machine-actionable optimization findings detected.")
+    for finding in findings:
+        savings = finding["savings"]
+        rewrite = finding["rewrite"]
+        validation = finding["validation"]
+        lines.extend(
+            [
+                f"### {markdown_heading(finding['finding_id'])}: "
+                f"{markdown_heading(finding['title'])}",
+                "",
+                f"- Severity: {markdown_text(finding['severity'])}",
+                f"- Type: {markdown_code_span(finding['type'])}",
+                f"- Confidence: {markdown_text(finding['confidence'])}",
+                "- Affected spans: "
+                f"{markdown_text(', '.join(map(str, finding['affected_spans'])) or 'none')}",
+                f"- Estimated latency savings: {format_duration_ms(savings['estimated_latency_savings_ms'])}",
+                "- Estimated cost savings: "
+                + format_cost_usd(savings.get("estimated_cost_savings_usd")),
+                f"- Rewrite: {markdown_text(rewrite['hint'])}",
+                f"- Validation: {markdown_text(validation['acceptance_criteria'])}",
+                "",
+            ]
+        )
+        if finding.get("rule_id"):
+            lines.append(
+                f"- Rule: {markdown_code_span(finding['rule_id'])} version {markdown_text(finding['rule_version'])}"
+            )
+        if finding.get("ranking"):
+            ranking = finding["ranking"]
+            lines.extend(
+                [
+                    f"- Investigation priority: {ranking['priority_rank']} ({markdown_text(ranking['status'])})",
+                    "- Ranking reasons: "
+                    + markdown_text(
+                        ", ".join(ranking["reasons"])
+                        or "Required inputs are complete; use the declared component order."
+                    ),
+                    "- Ranking components and provenance: "
+                    + markdown_code_span(json.dumps(ranking["components"], sort_keys=True)),
+                ]
+            )
+        if finding.get("evidence_level") and finding.get("estimate"):
+            lines.append(f"- Evidence level: {markdown_text(finding['evidence_level'])}")
+        lines.extend(estimate_markdown(finding.get("estimate")))
+        if finding.get("evidence_level") and not finding.get("estimate"):
+            lines.extend(
+                [
+                    f"- Evidence level: {markdown_text(finding['evidence_level'])}",
+                    "- Assumptions: "
+                    + "; ".join(markdown_text(value) for value in finding.get("assumptions", [])),
+                    f"- Savings formula: {markdown_text(savings.get('formula', ''))}",
+                    "",
+                ]
+            )
+    if "semantic_judgments" in diagnosis:
+        from agentloop.judgment_views import judgment_markdown
+
+        lines.extend(judgment_markdown(diagnosis["semantic_judgments"]))
+    if "semantic_waste" in diagnosis:
+        from agentloop.semantic_waste import semantic_waste_markdown
+
+        lines.extend(semantic_waste_markdown(diagnosis["semantic_waste"]))
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _stable_finding_id(
+    finding_type: str, title: str, affected_spans: list[str], *, context=None
+) -> str:
+    """Return an order-independent identity for one concrete detected finding.
+
+    The old IDs used a card enumeration index, so inserting or reordering an
+    unrelated card could make a human-reviewed status attach to different
+    evidence. The fingerprint is based on the finding kind, human-readable
+    title, and sorted affected span identities instead. A changed finding gets a
+    new ID; an unchanged finding keeps its ID regardless of card order.
+    """
+    identity = {
+        "type": finding_type,
+        "title": title,
+        "affected_spans": sorted(str(span) for span in affected_spans),
+    }
+    if context is not None:
+        identity["semantic_context"] = context
+    raw = json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    digest = hashlib.sha256(raw).hexdigest()[:12]
+    return f"al_{finding_type}_{digest}"
+
+
+def _finding_from_card(
+    card: dict[str, Any], plan: dict[str, Any], *, node_lookup=None
+) -> OptimizationFinding | None:
+    affected_spans = list(card.get("affected_nodes", []))
+    finding_type = str(card.get("type", "optimization"))
+    title = str(card.get("title", "Optimization opportunity"))
+    latency_savings = float(card.get("estimated_latency_savings_ms", 0.0) or 0.0)
+    raw_cost_savings = card.get("estimated_cost_savings_usd")
+    cost_savings = None if raw_cost_savings is None else float(raw_cost_savings)
+    severity = _severity(latency_savings, cost_savings, str(card.get("confidence", "low")), plan)
+    if node_lookup is None:
+        node_lookup = {node["node_id"]: node for node in plan.get("graph", {}).get("nodes", [])}
+    evidence = [
+        _evidence_row(node_lookup[node_id]) for node_id in affected_spans if node_id in node_lookup
+    ]
+    semantic = (card.get("estimate") or {}).get("estimator_id") == "semantic_leaf_removal"
+    observations = dict(card.get("observations", {}))
+    if semantic:
+        severity = "medium" if card.get("confidence") == "medium" else "low"
+        evidence.extend(
+            {"kind": "semantic_judgment", **item} for item in observations.get("judgments", [])
+        )
+
+    return OptimizationFinding(
+        finding_id=_stable_finding_id(
+            finding_type,
+            title,
+            affected_spans,
+            context={
+                "investigation_id": observations.get("investigation_id"),
+                "criteria_ref": observations.get("criteria_ref"),
+            }
+            if semantic
+            else None,
+        ),
+        severity=severity,
+        type=finding_type,
+        title=title,
+        confidence=str(card.get("confidence", "unknown")),
+        affected_spans=affected_spans,
+        evidence=evidence,
+        savings={
+            "estimated_latency_savings_ms": None
+            if card.get("estimated_latency_savings_ms") is None
+            else round(latency_savings, 3),
+            "estimated_cost_savings_usd": (
+                None if cost_savings is None else round(cost_savings, 6)
+            ),
+            "formula": (card.get("estimate") or {}).get("formula")
+            or card.get("estimate_formula")
+            or _savings_formula(finding_type),
+            **(
+                {
+                    "estimated_input_tokens": card["estimate"]["predictions"]["input_tokens"],
+                    "estimated_output_tokens": card["estimate"]["predictions"]["output_tokens"],
+                }
+                if semantic
+                else {}
+            ),
+        },
+        rewrite={
+            "kind": finding_type,
+            "hint": str(card.get("rewrite_hint", "")),
+            "patchable": finding_type
+            in {
+                "parallelize_tools",
+                "cache_context",
+                "add_schema_validation",
+                "batch_model_calls",
+                "route_to_smaller_model",
+                "split_large_step",
+                "runaway_loop",
+                "tool_oscillation",
+            },
+        },
+        validation={
+            "command": "agentloop replay",
+            "acceptance_criteria": observations["validation_criteria"]
+            if semantic
+            else _acceptance_criteria(finding_type),
+            **(
+                {"criteria_ref": observations["criteria_ref"], "requires_independent_quality": True}
+                if semantic
+                else {}
+            ),
+        },
+        metadata={
+            "why": card.get("why", ""),
+            "cost_status": "unknown"
+            if semantic and raw_cost_savings is None
+            else plan.get("cost_status"),
+            "identity": "semantic-context-v1" if semantic else "content-v1",
+        },
+        evidence_level=card.get("evidence_level"),
+        assumptions=list(card.get("assumptions", [])),
+        observations=observations,
+        rule_id=card.get("rule_id"),
+        rule_version=card.get("rule_version"),
+        estimate=deepcopy(card.get("estimate")),
+        ranking_requirements=tuple(card.get("ranking_requirements", ())),
+    )
+
+
+def _summary(plan: dict[str, Any], findings: list[OptimizationFinding]) -> dict[str, Any]:
+    return {
+        **{
+            key: plan["savings_aggregation"][key]
+            for key in ("unmodeled_latency_findings", "unmodeled_cost_findings")
+            if key in plan.get("savings_aggregation", {})
+        },
+        "finding_count": len(findings),
+        "high_severity_count": sum(1 for finding in findings if finding.severity == "high"),
+        "patchable_count": sum(1 for finding in findings if finding.rewrite.get("patchable")),
+        "estimated_latency_reduction_pct": plan["estimated_after"]["latency_reduction_pct"],
+        "estimated_cost_reduction_pct": plan["estimated_after"]["cost_reduction_pct"],
+    }
+
+
+def _severity(
+    latency_savings: float,
+    cost_savings: float | None,
+    confidence: str,
+    plan: dict[str, Any],
+) -> str:
+    runtime = float(plan.get("current", {}).get("runtime_ms", 0.0) or 0.0)
+    cost = float(plan.get("current", {}).get("estimated_cost_usd", 0.0) or 0.0)
+    latency_share = latency_savings / runtime if runtime else 0.0
+    cost_share = cost_savings / cost if cost and cost_savings is not None else 0.0
+    if confidence == "high" or latency_share >= 0.25 or cost_share >= 0.20:
+        return "high"
+    if confidence == "medium" or latency_share >= 0.10 or cost_share >= 0.10:
+        return "medium"
+    return "low"
+
+
+def _evidence_row(node: dict[str, Any]) -> dict[str, Any]:
+    return {
+        **({"stage": deepcopy(node["stage"])} if "stage" in node else {}),
+        "span_id": node["node_id"],
+        "name": node["name"],
+        "event_type": node["event_type"],
+        "operation_kind": node.get("operation_kind", "unknown"),
+        "duration_ms": node["duration_ms"],
+        "input_tokens": node.get("input_tokens", 0),
+        "output_tokens": node.get("output_tokens", 0),
+        "model": node.get("model"),
+        "status": node.get("status", "ok"),
+    }
+
+
+def _savings_formula(finding_type: str) -> str:
+    formulas = {
+        "parallelize_tools": "sum(serial tool durations) - max(tool duration)",
+        "cache_context": "current cost * repeated context ratio, capped at 50%",
+        "add_schema_validation": "retry duration * 80%",
+        "batch_model_calls": "sum(repeated model durations) * 35%",
+        "route_to_smaller_model": "model step duration * 25%",
+        "split_large_step": "large step duration * 20%",
+        "runaway_loop": "repeated loop duration * 30%",
+        "tool_oscillation": "alternating tool duration * 40%",
+    }
+    return formulas.get(finding_type, "optimizer estimate")
+
+
+def _acceptance_criteria(finding_type: str) -> str:
+    criteria = {
+        "parallelize_tools": "runtime decreases while tool outputs remain equivalent",
+        "cache_context": "input tokens and cost decrease without output quality regression",
+        "add_schema_validation": "retry count decreases and output schema validity stays at 100%",
+        "batch_model_calls": "runtime decreases and item-level outputs remain equivalent",
+        "route_to_smaller_model": "cost decreases while configured quality scorer passes",
+        "split_large_step": "context size decreases while final answer quality scorer passes",
+        "runaway_loop": "iteration count is bounded and replay passes quality and budget gates",
+        "tool_oscillation": "tool-call count decreases without losing required state transitions",
+    }
+    return criteria.get(
+        finding_type, "before/after replay passes configured quality and budget gates"
+    )