agentloop-profiler 0.4.0__py3-none-any.whl

agentloop/doctor.py

@@ -0,0 +1,344 @@
+from __future__ import annotations
+
+import importlib
+import os
+import platform
+import socket
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.error import URLError
+from urllib.parse import urlsplit
+from urllib.request import Request, urlopen
+
+from agentloop.config import postgres_connection_source
+from agentloop.runtime import get_runtime_config
+from agentloop.store import get_store
+
+
+@dataclass
+class DoctorCheck:
+    name: str
+    status: str
+    detail: str
+    fix: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = {"name": self.name, "status": self.status, "detail": self.detail}
+        if self.fix:
+            payload["fix"] = self.fix
+        return payload
+
+
+def _ok(name: str, detail: str) -> DoctorCheck:
+    return DoctorCheck(name=name, status="ok", detail=detail)
+
+
+def _warn(name: str, detail: str, fix: str | None = None) -> DoctorCheck:
+    return DoctorCheck(name=name, status="warn", detail=detail, fix=fix)
+
+
+def _fail(name: str, detail: str, fix: str | None = None) -> DoctorCheck:
+    return DoctorCheck(name=name, status="fail", detail=detail, fix=fix)
+
+
+def _can_connect(host: str, port: int, timeout: float = 1.0) -> bool:
+    try:
+        with socket.create_connection((host, port), timeout=timeout):
+            return True
+    except OSError:
+        return False
+
+
+def _health_check(api_url: str, timeout: float = 2.0) -> DoctorCheck:
+    parsed = urlsplit(api_url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        return _warn(
+            "api_health",
+            "AGENTLOOP_API_URL must use http:// or https://",
+            "Set AGENTLOOP_API_URL to the AgentLoop HTTP API origin.",
+        )
+    url = api_url.rstrip("/") + "/health"
+    try:
+        request = Request(url, headers={"User-Agent": "agentloop-doctor"})
+        with urlopen(  # nosec B310 - API URL scheme is validated above.
+            request, timeout=timeout
+        ) as response:
+            if response.status == 200:
+                return _ok("api_health", f"{url} returned HTTP 200")
+            return _warn("api_health", f"{url} returned HTTP {response.status}")
+    except URLError as exc:
+        return _warn(
+            "api_health",
+            f"Could not reach {url}: {exc.reason}",
+            "Start the API with: agentloop server --host 127.0.0.1 --port 8000",
+        )
+    except Exception as exc:  # pragma: no cover - defensive for platform SSL/socket edge cases
+        return _warn("api_health", f"Could not reach {url}: {exc}")
+
+
+def _ready_check(api_url: str, timeout: float = 2.0) -> DoctorCheck:
+    parsed = urlsplit(api_url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        return _fail(
+            "api_ready",
+            "AGENTLOOP_API_URL must use http:// or https://",
+            "Set AGENTLOOP_API_URL to the deployed AgentLoop HTTP API origin.",
+        )
+    url = api_url.rstrip("/") + "/readyz"
+    try:
+        request = Request(url, headers={"User-Agent": "agentloop-production-check"})
+        with urlopen(  # nosec B310 - API URL scheme is validated above.
+            request, timeout=timeout
+        ) as response:
+            if response.status == 200:
+                return _ok("api_ready", f"{url} returned HTTP 200")
+            return _fail("api_ready", f"{url} returned HTTP {response.status}")
+    except URLError as exc:
+        return _fail(
+            "api_ready",
+            f"Could not reach {url}: {exc.reason}",
+            "Deploy the API and confirm /readyz is reachable.",
+        )
+    except Exception as exc:  # pragma: no cover - defensive for platform SSL/socket edge cases
+        return _fail("api_ready", f"Could not reach {url}: {exc}")
+
+
+def _env_enabled(name: str) -> bool:
+    return os.getenv(name, "").lower() in {"1", "true", "yes", "on"}
+
+
+def run_doctor(*, check_api: bool = True, runs_dir: Path | str = "runs") -> dict[str, Any]:
+    """Run local setup checks for SDK, CLI, store, and hosted API wiring."""
+
+    checks: list[DoctorCheck] = []
+
+    py_version = sys.version_info
+    if py_version >= (3, 10):
+        checks.append(_ok("python", f"Python {platform.python_version()}"))
+    else:
+        checks.append(
+            _fail(
+                "python",
+                f"Python {platform.python_version()} is too old",
+                "Use Python 3.10 or newer.",
+            )
+        )
+
+    for module_name in ["typer", "rich", "pydantic"]:
+        try:
+            importlib.import_module(module_name)
+            checks.append(_ok(f"dependency:{module_name}", "importable"))
+        except ImportError:
+            checks.append(
+                _fail(
+                    f"dependency:{module_name}",
+                    "not installed",
+                    "Run: uv sync --locked --all-extras --dev",
+                )
+            )
+
+    out_dir = Path(runs_dir)
+    try:
+        out_dir.mkdir(parents=True, exist_ok=True)
+        probe = out_dir / ".agentloop_write_probe"
+        probe.write_text("ok", encoding="utf-8")
+        probe.unlink(missing_ok=True)
+        checks.append(_ok("runs_dir", f"{out_dir} is writable"))
+    except Exception as exc:
+        checks.append(
+            _fail(
+                "runs_dir",
+                f"{out_dir} is not writable: {exc}",
+                "Choose a writable path or set AGENTLOOP_EXPORT_DIR.",
+            )
+        )
+
+    try:
+        db = get_store()
+        db.init()
+        checks.append(_ok("store", f"{db.__class__.__name__} initialized"))
+    except Exception as exc:
+        checks.append(
+            _fail(
+                "store",
+                f"store initialization failed: {exc}",
+                "Check AGENTLOOP_STORE_BACKEND and database credentials.",
+            )
+        )
+
+    cfg = get_runtime_config()
+    if (
+        cfg.auto_upload
+        and not cfg.api_key
+        and os.getenv("AGENTLOOP_REQUIRE_API_KEY", "").lower() in {"1", "true", "yes", "on"}
+    ):
+        checks.append(
+            _warn(
+                "api_key",
+                "auto upload is enabled but AGENTLOOP_API_KEY is empty",
+                "Create a key with: agentloop create-api-key --project-id demo",
+            )
+        )
+    elif cfg.api_key:
+        checks.append(_ok("api_key", "AGENTLOOP_API_KEY is configured"))
+    else:
+        checks.append(
+            _warn("api_key", "no API key configured; OK for local API when auth is disabled")
+        )
+
+    if cfg.auto_upload:
+        checks.append(_ok("auto_upload", f"enabled -> {cfg.api_url}"))
+    else:
+        checks.append(
+            _warn(
+                "auto_upload",
+                "disabled",
+                "Set AGENTLOOP_AUTO_UPLOAD=true to upload traces automatically.",
+            )
+        )
+
+    if check_api:
+        checks.append(_health_check(cfg.api_url))
+
+    failed = sum(1 for check in checks if check.status == "fail")
+    warned = sum(1 for check in checks if check.status == "warn")
+    return {
+        "ok": failed == 0,
+        "failed": failed,
+        "warnings": warned,
+        "checks": [check.to_dict() for check in checks],
+    }
+
+
+def run_production_check(
+    *,
+    check_api: bool = True,
+    check_store: bool = True,
+    allow_http: bool = False,
+) -> dict[str, Any]:
+    """Run strict checks for an internet-facing AgentLoop API/dashboard deployment."""
+
+    checks: list[DoctorCheck] = []
+
+    py_version = sys.version_info
+    if py_version >= (3, 10):
+        checks.append(_ok("python", f"Python {platform.python_version()}"))
+    else:
+        checks.append(
+            _fail(
+                "python",
+                f"Python {platform.python_version()} is too old",
+                "Use Python 3.10 or newer.",
+            )
+        )
+
+    backend = os.getenv("AGENTLOOP_STORE_BACKEND", "sqlite").lower()
+    if backend == "postgres":
+        checks.append(_ok("store_backend", "postgres"))
+    else:
+        checks.append(
+            _fail(
+                "store_backend",
+                f"{backend} is not production-ready",
+                "Set AGENTLOOP_STORE_BACKEND=postgres.",
+            )
+        )
+
+    database_source = postgres_connection_source()
+    if database_source:
+        checks.append(_ok("database_url", f"configured via {database_source}"))
+    else:
+        checks.append(
+            _fail(
+                "database_url",
+                "missing",
+                "Set AGENTLOOP_DATABASE_URL/DATABASE_URL or libpq PGHOST/PGSERVICE variables.",
+            )
+        )
+
+    if check_store:
+        try:
+            db = get_store()
+            db.init()
+            checks.append(_ok("store", f"{db.__class__.__name__} initialized"))
+        except Exception as exc:
+            checks.append(
+                _fail(
+                    "store",
+                    f"store initialization failed: {exc}",
+                    "Check database credentials and network access.",
+                )
+            )
+
+    if _env_enabled("AGENTLOOP_REQUIRE_API_KEY"):
+        checks.append(_ok("api_auth", "AGENTLOOP_REQUIRE_API_KEY=true"))
+    else:
+        checks.append(
+            _fail("api_auth", "API key auth is disabled", "Set AGENTLOOP_REQUIRE_API_KEY=true.")
+        )
+
+    admin_key = os.getenv("AGENTLOOP_ADMIN_API_KEY", "")
+    weak_admin_values = {
+        "",
+        "change-me-admin-key",
+        "replace-with-a-long-random-admin-secret",
+        "dev-secret",
+        "admin-secret",
+    }
+    if admin_key in weak_admin_values or len(admin_key) < 32:
+        checks.append(
+            _fail(
+                "admin_api_key",
+                "missing or too weak",
+                "Set AGENTLOOP_ADMIN_API_KEY to a generated secret with at least 32 characters.",
+            )
+        )
+    else:
+        checks.append(_ok("admin_api_key", "configured"))
+
+    cors_origins = [
+        origin.strip()
+        for origin in os.getenv("AGENTLOOP_CORS_ORIGINS", "").split(",")
+        if origin.strip()
+    ]
+    if not cors_origins:
+        checks.append(
+            _fail("cors_origins", "missing", "Set AGENTLOOP_CORS_ORIGINS to the dashboard origin.")
+        )
+    elif "*" in cors_origins:
+        checks.append(
+            _fail(
+                "cors_origins",
+                "wildcard origin is not allowed for production",
+                "Use exact HTTPS origins.",
+            )
+        )
+    else:
+        checks.append(_ok("cors_origins", ", ".join(cors_origins)))
+
+    api_url = os.getenv("AGENTLOOP_API_URL", "")
+    if not api_url:
+        checks.append(_fail("api_url", "missing", "Set AGENTLOOP_API_URL to the public API URL."))
+    elif api_url.startswith("https://"):
+        checks.append(_ok("api_url", api_url))
+    elif allow_http:
+        checks.append(
+            _warn("api_url", f"{api_url} is not HTTPS", "Use HTTPS for production traffic.")
+        )
+    else:
+        checks.append(_fail("api_url", f"{api_url} is not HTTPS", "Use an HTTPS public API URL."))
+
+    if check_api and api_url:
+        checks.append(_health_check(api_url))
+        checks.append(_ready_check(api_url))
+
+    failed = sum(1 for check in checks if check.status == "fail")
+    warned = sum(1 for check in checks if check.status == "warn")
+    return {
+        "ok": failed == 0,
+        "failed": failed,
+        "warnings": warned,
+        "checks": [check.to_dict() for check in checks],
+    }

agentloop/events.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def new_event_id() -> str:
+    return "evt_" + uuid4().hex[:16]
+
+
+def new_run_id() -> str:
+    return "run_" + uuid4().hex[:16]
+
+
+@dataclass
+class AgentEvent:
+    event_id: str
+    run_id: str
+    event_type: str
+    name: str
+    started_at: str
+    ended_at: str
+    duration_ms: float
+    parent_id: str | None = None
+    model: str | None = None
+    input_tokens: int = 0
+    output_tokens: int = 0
+    input_text: str | None = None
+    output_text: str | None = None
+    status: str = "ok"
+    error: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def total_tokens(self) -> int:
+        return self.input_tokens + self.output_tokens
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.__dict__.copy()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "AgentEvent":
+        return cls(**data)

agentloop/exporters.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from agentloop.markdown import (
+    markdown_code_span,
+    markdown_heading,
+    markdown_table_cell,
+    markdown_text,
+)
+
+
+def export_report_json(report: dict[str, Any], path: str | Path) -> Path:
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(report, indent=2), encoding="utf-8")
+    return out
+
+
+def export_report_markdown(report: dict[str, Any], path: str | Path) -> Path:
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    lines = [
+        f"# AgentLoop Report: {markdown_heading(report['name'])}",
+        "",
+        f"- Run ID: {markdown_code_span(report['run_id'])}",
+        f"- Runtime: {report['total_runtime_ms'] / 1000:.2f}s",
+        f"- Estimated cost: ${report['estimated_cost_usd']:.4f}",
+        f"- Model calls: {report['model_call_count']}",
+        f"- Tool calls: {report['tool_call_count']}",
+        f"- Retries: {report['retry_count']}",
+        f"- Input tokens: {report['input_tokens']}",
+        f"- Output tokens: {report['output_tokens']}",
+        f"- Repeated context: {report['repeated_context_ratio']:.1%}",
+        "",
+        "## Recommendations",
+    ]
+    for rec in report["recommendations"]:
+        lines.append(f"- **{markdown_text(rec['title'])}** — {markdown_text(rec['description'])}")
+    lines.extend(
+        [
+            "",
+            "## Events",
+            "",
+            "| Type | Name | Duration ms | Model | Input | Output | Status |",
+            "|---|---|---:|---|---:|---:|---|",
+        ]
+    )
+    for event in report["events"]:
+        lines.append(
+            f"| {markdown_table_cell(event['event_type'])} | "
+            f"{markdown_table_cell(event['name'])} | {event['duration_ms']:.2f} | "
+            f"{markdown_table_cell(event.get('model') or '')} | "
+            f"{event.get('input_tokens', 0)} | {event.get('output_tokens', 0)} | "
+            f"{markdown_table_cell(event.get('status', 'ok'))} |"
+        )
+    out.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return out

agentloop/findings.py

@@ -0,0 +1,215 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from agentloop.markdown import markdown_code_span, markdown_heading, markdown_text
+from agentloop.optimizer import build_optimization_plan
+
+
+@dataclass
+class OptimizationFinding:
+    finding_id: str
+    severity: str
+    type: str
+    title: str
+    confidence: str
+    affected_spans: list[str]
+    evidence: list[dict[str, Any]]
+    savings: dict[str, Any]
+    rewrite: dict[str, Any]
+    validation: dict[str, Any]
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "finding_id": self.finding_id,
+            "severity": self.severity,
+            "type": self.type,
+            "title": self.title,
+            "confidence": self.confidence,
+            "affected_spans": self.affected_spans,
+            "evidence": self.evidence,
+            "savings": self.savings,
+            "rewrite": self.rewrite,
+            "validation": self.validation,
+            "metadata": self.metadata,
+        }
+
+
+def build_diagnosis(trace: Any) -> dict[str, Any]:
+    plan = build_optimization_plan(trace)
+    findings = [
+        _finding_from_card(index, card, plan)
+        for index, card in enumerate(plan.get("optimization_cards", []), start=1)
+    ]
+    findings = [finding for finding in findings if finding is not None]
+    return {
+        "run_id": plan["run_id"],
+        "name": plan["name"],
+        "current": plan["current"],
+        "estimated_after": plan["estimated_after"],
+        "summary": _summary(plan, findings),
+        "findings": [finding.to_dict() for finding in findings],
+        "graph": plan["graph"],
+    }
+
+
+def diagnosis_to_markdown(diagnosis: dict[str, Any]) -> str:
+    current = diagnosis["current"]
+    after = diagnosis["estimated_after"]
+    lines = [
+        f"# AgentLoop Diagnosis: {markdown_heading(diagnosis['name'])}",
+        "",
+        f"- Run ID: {markdown_code_span(diagnosis['run_id'])}",
+        f"- Current runtime: {current['runtime_ms'] / 1000:.2f}s",
+        f"- Estimated runtime after fixes: {after['runtime_ms'] / 1000:.2f}s",
+        f"- Estimated latency reduction: {after['latency_reduction_pct']:.2f}%",
+        f"- Current cost: ${current['estimated_cost_usd']:.4f}",
+        f"- Estimated cost after fixes: ${after['estimated_cost_usd']:.4f}",
+        f"- Estimated cost reduction: {after['cost_reduction_pct']:.2f}%",
+        "",
+        "## Findings",
+        "",
+    ]
+    findings = diagnosis.get("findings", [])
+    if not findings:
+        lines.append("No machine-actionable optimization findings detected.")
+    for finding in findings:
+        savings = finding["savings"]
+        rewrite = finding["rewrite"]
+        validation = finding["validation"]
+        lines.extend(
+            [
+                f"### {markdown_heading(finding['finding_id'])}: "
+                f"{markdown_heading(finding['title'])}",
+                "",
+                f"- Severity: {markdown_text(finding['severity'])}",
+                f"- Type: {markdown_code_span(finding['type'])}",
+                f"- Confidence: {markdown_text(finding['confidence'])}",
+                "- Affected spans: "
+                f"{markdown_text(', '.join(map(str, finding['affected_spans'])) or 'none')}",
+                f"- Estimated latency savings: {savings['estimated_latency_savings_ms'] / 1000:.2f}s",
+                f"- Estimated cost savings: ${savings['estimated_cost_savings_usd']:.4f}",
+                f"- Rewrite: {markdown_text(rewrite['hint'])}",
+                f"- Validation: {markdown_text(validation['acceptance_criteria'])}",
+                "",
+            ]
+        )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _finding_from_card(
+    index: int, card: dict[str, Any], plan: dict[str, Any]
+) -> OptimizationFinding | None:
+    affected_spans = list(card.get("affected_nodes", []))
+    finding_type = str(card.get("type", "optimization"))
+    latency_savings = float(card.get("estimated_latency_savings_ms", 0.0) or 0.0)
+    cost_savings = float(card.get("estimated_cost_savings_usd", 0.0) or 0.0)
+    severity = _severity(latency_savings, cost_savings, str(card.get("confidence", "low")), plan)
+    node_lookup = {node["node_id"]: node for node in plan.get("graph", {}).get("nodes", [])}
+    evidence = [
+        _evidence_row(node_lookup[node_id]) for node_id in affected_spans if node_id in node_lookup
+    ]
+
+    return OptimizationFinding(
+        finding_id=f"al_{finding_type}_{index:03d}",
+        severity=severity,
+        type=finding_type,
+        title=str(card.get("title", "Optimization opportunity")),
+        confidence=str(card.get("confidence", "unknown")),
+        affected_spans=affected_spans,
+        evidence=evidence,
+        savings={
+            "estimated_latency_savings_ms": round(latency_savings, 3),
+            "estimated_cost_savings_usd": round(cost_savings, 6),
+            "formula": _savings_formula(finding_type),
+        },
+        rewrite={
+            "kind": finding_type,
+            "hint": str(card.get("rewrite_hint", "")),
+            "patchable": finding_type
+            in {
+                "parallelize_tools",
+                "cache_context",
+                "add_schema_validation",
+                "batch_model_calls",
+                "route_to_smaller_model",
+                "split_large_step",
+                "runaway_loop",
+                "tool_oscillation",
+            },
+        },
+        validation={
+            "command": "agentloop replay",
+            "acceptance_criteria": _acceptance_criteria(finding_type),
+        },
+        metadata={"why": card.get("why", "")},
+    )
+
+
+def _summary(plan: dict[str, Any], findings: list[OptimizationFinding]) -> dict[str, Any]:
+    return {
+        "finding_count": len(findings),
+        "high_severity_count": sum(1 for finding in findings if finding.severity == "high"),
+        "patchable_count": sum(1 for finding in findings if finding.rewrite.get("patchable")),
+        "estimated_latency_reduction_pct": plan["estimated_after"]["latency_reduction_pct"],
+        "estimated_cost_reduction_pct": plan["estimated_after"]["cost_reduction_pct"],
+    }
+
+
+def _severity(
+    latency_savings: float, cost_savings: float, confidence: str, plan: dict[str, Any]
+) -> str:
+    runtime = float(plan.get("current", {}).get("runtime_ms", 0.0) or 0.0)
+    cost = float(plan.get("current", {}).get("estimated_cost_usd", 0.0) or 0.0)
+    latency_share = latency_savings / runtime if runtime else 0.0
+    cost_share = cost_savings / cost if cost else 0.0
+    if confidence == "high" or latency_share >= 0.25 or cost_share >= 0.20:
+        return "high"
+    if confidence == "medium" or latency_share >= 0.10 or cost_share >= 0.10:
+        return "medium"
+    return "low"
+
+
+def _evidence_row(node: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "span_id": node["node_id"],
+        "name": node["name"],
+        "event_type": node["event_type"],
+        "duration_ms": node["duration_ms"],
+        "input_tokens": node.get("input_tokens", 0),
+        "output_tokens": node.get("output_tokens", 0),
+        "model": node.get("model"),
+        "status": node.get("status", "ok"),
+    }
+
+
+def _savings_formula(finding_type: str) -> str:
+    formulas = {
+        "parallelize_tools": "sum(serial tool durations) - max(tool duration)",
+        "cache_context": "current cost * repeated context ratio, capped at 50%",
+        "add_schema_validation": "retry duration * 80%",
+        "batch_model_calls": "sum(repeated model durations) * 35%",
+        "route_to_smaller_model": "model step duration * 25%",
+        "split_large_step": "large step duration * 20%",
+        "runaway_loop": "repeated loop duration * 30%",
+        "tool_oscillation": "alternating tool duration * 40%",
+    }
+    return formulas.get(finding_type, "optimizer estimate")
+
+
+def _acceptance_criteria(finding_type: str) -> str:
+    criteria = {
+        "parallelize_tools": "runtime decreases while tool outputs remain equivalent",
+        "cache_context": "input tokens and cost decrease without output quality regression",
+        "add_schema_validation": "retry count decreases and output schema validity stays at 100%",
+        "batch_model_calls": "runtime decreases and item-level outputs remain equivalent",
+        "route_to_smaller_model": "cost decreases while configured quality scorer passes",
+        "split_large_step": "context size decreases while final answer quality scorer passes",
+        "runaway_loop": "iteration count is bounded and replay passes quality and budget gates",
+        "tool_oscillation": "tool-call count decreases without losing required state transitions",
+    }
+    return criteria.get(
+        finding_type, "before/after replay passes configured quality and budget gates"
+    )