agentloop-profiler 0.4.0__py3-none-any.whl

agentloop/savings.py

@@ -0,0 +1,135 @@
+"""Compatible-selection model for combining per-recommendation savings.
+
+Optimization cards (and the findings derived from them) that touch the same
+spans compete for the same work: applying one changes or removes the spans the
+other targeted, so their estimates are alternatives, not additive. A realizable
+plan therefore applies a pairwise span-disjoint subset of the recommendations.
+
+``select_compatible`` picks that subset: latency savings is the primary
+objective and cost savings breaks ties, so both reported totals always come
+from the same subset — the (latency, cost) pair is achievable by one concrete
+plan rather than a mix of incompatible alternatives. Items without affected
+spans do not compete with anything and always count.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterator, Sequence
+
+# Beyond this many mutually overlapping items the exact search (exponential in
+# the size of the overlap-connected component) falls back to a greedy pick.
+_EXACT_SEARCH_LIMIT = 16
+
+
+@dataclass(frozen=True)
+class SavingsItem:
+    spans: frozenset[str]
+    latency_ms: float = 0.0
+    cost_usd: float = 0.0
+
+
+@dataclass(frozen=True)
+class Selection:
+    indices: tuple[int, ...]
+    latency_ms: float
+    cost_usd: float
+
+
+def select_compatible(items: Sequence[SavingsItem]) -> Selection:
+    """Return the compatible (span-disjoint) subset with the best savings.
+
+    Maximizes total latency savings, breaking ties by total cost savings.
+    Span-free items are always selected.
+    """
+    chosen = [index for index, item in enumerate(items) if not item.spans]
+    spanned = [index for index, item in enumerate(items) if item.spans]
+    neighbor_sets: dict[int, set[int]] = {index: set() for index in spanned}
+    for position, left in enumerate(spanned):
+        for right in spanned[position + 1 :]:
+            if items[left].spans & items[right].spans:
+                neighbor_sets[left].add(right)
+                neighbor_sets[right].add(left)
+    adjacency = {index: frozenset(neighbors) for index, neighbors in neighbor_sets.items()}
+    for component in _components(spanned, adjacency):
+        if len(component) > _EXACT_SEARCH_LIMIT:
+            chosen.extend(_greedy_pick(items, component))
+        else:
+            chosen.extend(_exact_pick(items, component, adjacency))
+    chosen.sort()
+    return Selection(
+        indices=tuple(chosen),
+        latency_ms=sum(items[index].latency_ms for index in chosen),
+        cost_usd=sum(items[index].cost_usd for index in chosen),
+    )
+
+
+def _components(nodes: list[int], adjacency: dict[int, frozenset[int]]) -> Iterator[list[int]]:
+    seen: set[int] = set()
+    for start in nodes:
+        if start in seen:
+            continue
+        seen.add(start)
+        stack = [start]
+        component: list[int] = []
+        while stack:
+            node = stack.pop()
+            component.append(node)
+            for neighbor in adjacency[node]:
+                if neighbor not in seen:
+                    seen.add(neighbor)
+                    stack.append(neighbor)
+        yield component
+
+
+def _savings_key(items: Sequence[SavingsItem], index: int) -> tuple[float, float]:
+    return (items[index].latency_ms, items[index].cost_usd)
+
+
+def _exact_pick(
+    items: Sequence[SavingsItem], component: list[int], adjacency: dict[int, frozenset[int]]
+) -> list[int]:
+    order = sorted(component, key=lambda index: _savings_key(items, index), reverse=True)
+    suffix_latency = [0.0] * (len(order) + 1)
+    for position in range(len(order) - 1, -1, -1):
+        suffix_latency[position] = suffix_latency[position + 1] + items[order[position]].latency_ms
+    best_key = (float("-inf"), float("-inf"))
+    best: list[int] = []
+
+    def search(
+        position: int, blocked: frozenset[int], latency: float, cost: float, picked: list[int]
+    ) -> None:
+        nonlocal best_key, best
+        if latency + suffix_latency[position] < best_key[0]:
+            return
+        if position == len(order):
+            if (latency, cost) > best_key:
+                best_key = (latency, cost)
+                best = list(picked)
+            return
+        index = order[position]
+        if index not in blocked:
+            picked.append(index)
+            search(
+                position + 1,
+                blocked | adjacency[index],
+                latency + items[index].latency_ms,
+                cost + items[index].cost_usd,
+                picked,
+            )
+            picked.pop()
+        search(position + 1, blocked, latency, cost, picked)
+
+    search(0, frozenset(), 0.0, 0.0, [])
+    return best
+
+
+def _greedy_pick(items: Sequence[SavingsItem], component: list[int]) -> list[int]:
+    picked: list[int] = []
+    used_spans: set[str] = set()
+    for index in sorted(component, key=lambda index: _savings_key(items, index), reverse=True):
+        if items[index].spans & used_spans:
+            continue
+        picked.append(index)
+        used_spans |= items[index].spans
+    return picked

agentloop/server.py

@@ -0,0 +1,292 @@
+from __future__ import annotations
+
+import hmac
+from typing import Any
+
+from fastapi import Depends, FastAPI, Header, HTTPException, Query
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field
+
+from agentloop.config import get_admin_api_key, get_api_key, get_cors_origins, require_api_key
+from agentloop.findings import build_diagnosis
+from agentloop.issues import build_issue_drafts
+from agentloop.optimizer import build_optimization_plan
+from agentloop.quality import build_quality_report
+from agentloop.store import TraceProjectConflictError, TraceStore, get_store
+from agentloop.tracer import AgentTrace
+from agentloop.value import build_value_report
+from agentloop.version import __version__
+
+app = FastAPI(title="AgentLoop API", version=__version__)
+
+_cors_origins = get_cors_origins()
+if _cors_origins:
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=_cors_origins,
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+
+class TracePayload(BaseModel):
+    name: str
+    run_id: str
+    started_at: str | None = None
+    ended_at: str | None = None
+    elapsed_ms: float | None = Field(default=None, ge=0)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    events: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class CreateApiKeyPayload(BaseModel):
+    project_id: str = "default"
+    name: str = "default"
+
+
+class QualityReportPayload(BaseModel):
+    fixtures: list[dict[str, Any]] = Field(default_factory=list)
+    baseline_run_id: str | None = None
+    candidate_run_id: str | None = None
+    min_score: float | None = Field(default=None, ge=0)
+
+
+def store() -> TraceStore:
+    db = get_store()
+    db.init()
+    return db
+
+
+def resolve_project(
+    x_agentloop_key: str | None = Header(default=None),
+    db: TraceStore = Depends(store),
+) -> str:
+    if x_agentloop_key:
+        key_record = db.verify_api_key(x_agentloop_key)
+        if key_record is not None:
+            return str(key_record["project_id"])
+
+        expected = get_api_key()
+        if require_api_key() and expected and hmac.compare_digest(x_agentloop_key, expected):
+            return "default"
+
+        raise HTTPException(status_code=401, detail="invalid AgentLoop API key")
+
+    if require_api_key():
+        raise HTTPException(status_code=401, detail="missing AgentLoop API key")
+
+    return "default"
+
+
+def resolve_admin(x_agentloop_admin_key: str | None = Header(default=None)) -> None:
+    admin_key = get_admin_api_key()
+    if admin_key:
+        if x_agentloop_admin_key and hmac.compare_digest(x_agentloop_admin_key, admin_key):
+            return
+        raise HTTPException(status_code=401, detail="invalid AgentLoop admin API key")
+
+    if require_api_key():
+        raise HTTPException(
+            status_code=503,
+            detail="AGENTLOOP_ADMIN_API_KEY is required to create API keys when API auth is enabled",
+        )
+
+
+@app.get("/health")
+def health() -> dict[str, str | bool]:
+    return {"status": "ok", "version": __version__, "auth_required": require_api_key()}
+
+
+@app.get("/readyz")
+def readyz(db: TraceStore = Depends(store)) -> dict[str, str]:
+    db.usage_summary(project_id="default")
+    return {"status": "ready"}
+
+
+@app.post("/api-keys")
+def create_api_key(
+    payload: CreateApiKeyPayload,
+    _: None = Depends(resolve_admin),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    return db.create_api_key(project_id=payload.project_id, name=payload.name)
+
+
+@app.post("/traces")
+def ingest_trace(
+    payload: TracePayload,
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    trace = AgentTrace.from_dict(payload.model_dump())
+    try:
+        db.save_trace(trace, project_id=project_id)
+    except TraceProjectConflictError as exc:
+        raise HTTPException(status_code=409, detail=str(exc)) from None
+    return {"ok": True, "run_id": trace.run_id, "project_id": project_id, "report": trace.report()}
+
+
+def _selected_project(project_id_filter: str | None, authenticated_project: str) -> str:
+    if (
+        require_api_key()
+        and project_id_filter is not None
+        and project_id_filter != authenticated_project
+    ):
+        raise HTTPException(status_code=403, detail="project access denied")
+    return project_id_filter or authenticated_project
+
+
+@app.get("/traces")
+def list_traces(
+    project_id_filter: str | None = Query(default=None, alias="project_id"),
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    selected_project = _selected_project(project_id_filter, project_id)
+    return {"project_id": selected_project, "traces": db.list_traces(project_id=selected_project)}
+
+
+def _load_trace_or_404(db: TraceStore, run_id: str, project_id: str) -> AgentTrace:
+    trace = db.get_trace(run_id=run_id, project_id=project_id)
+    if trace is None:
+        raise HTTPException(status_code=404, detail="trace not found")
+    return trace
+
+
+@app.get("/traces/{run_id}/report")
+def get_report(
+    run_id: str,
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    return _load_trace_or_404(db, run_id, project_id).report()
+
+
+@app.get("/traces/{run_id}/optimize")
+def optimize_trace(
+    run_id: str,
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    return build_optimization_plan(_load_trace_or_404(db, run_id, project_id))
+
+
+@app.get("/traces/{run_id}/diagnose")
+def diagnose_trace(
+    run_id: str,
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    diagnosis = build_diagnosis(_load_trace_or_404(db, run_id, project_id))
+    db.save_diagnosis(diagnosis, project_id=project_id)
+    return diagnosis
+
+
+@app.get("/findings")
+def list_findings(
+    status: str | None = Query(default=None),
+    project_id_filter: str | None = Query(default=None, alias="project_id"),
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    selected_project = _selected_project(project_id_filter, project_id)
+    return {
+        "project_id": selected_project,
+        "findings": db.list_findings(project_id=selected_project, status=status),
+    }
+
+
+@app.get("/optimization-queue")
+def optimization_queue(
+    project_id_filter: str | None = Query(default=None, alias="project_id"),
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    selected_project = _selected_project(project_id_filter, project_id)
+    return {
+        "project_id": selected_project,
+        "queue": db.optimization_queue(project_id=selected_project),
+    }
+
+
+@app.get("/optimization-queue/github-issues")
+def github_issue_drafts(
+    limit: int = Query(default=5, ge=1, le=20),
+    project_id_filter: str | None = Query(default=None, alias="project_id"),
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    selected_project = _selected_project(project_id_filter, project_id)
+    queue = db.optimization_queue(project_id=selected_project)
+    return {
+        "project_id": selected_project,
+        "issue_drafts": build_issue_drafts(queue, limit=limit),
+    }
+
+
+@app.post("/quality-report")
+def quality_report_endpoint(
+    payload: QualityReportPayload,
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    scorer_types = {
+        str(fixture["scorer"].get("type", "")).lower()
+        for fixture in payload.fixtures
+        if isinstance(fixture.get("scorer"), dict)
+    }
+    if "custom" in scorer_types:
+        raise HTTPException(
+            status_code=400,
+            detail="custom Python scorers are not accepted by the HTTP API",
+        )
+    if "regex" in scorer_types:
+        raise HTTPException(
+            status_code=400,
+            detail="raw regex scorers are not accepted; use glob, contains, or exact_match",
+        )
+    baseline_trace = (
+        _load_trace_or_404(db, payload.baseline_run_id, project_id)
+        if payload.baseline_run_id
+        else None
+    )
+    candidate_trace = (
+        _load_trace_or_404(db, payload.candidate_run_id, project_id)
+        if payload.candidate_run_id
+        else None
+    )
+    return build_quality_report(
+        payload.fixtures,
+        baseline_trace=baseline_trace,
+        candidate_trace=candidate_trace,
+        min_score=payload.min_score,
+    )
+
+
+@app.get("/traces/{run_id}/value")
+def value_report(
+    run_id: str,
+    runs_per_month: int = Query(default=1000, ge=0),
+    engineer_hourly_rate_usd: float = Query(default=150.0, ge=0),
+    incident_cost_usd: float = Query(default=500.0, ge=0),
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    trace = _load_trace_or_404(db, run_id, project_id)
+    return build_value_report(
+        trace,
+        runs_per_month=runs_per_month,
+        engineer_hourly_rate_usd=engineer_hourly_rate_usd,
+        incident_cost_usd=incident_cost_usd,
+    )
+
+
+@app.get("/usage")
+def usage_summary(
+    project_id_filter: str | None = Query(default=None, alias="project_id"),
+    project_id: str = Depends(resolve_project),
+    db: TraceStore = Depends(store),
+) -> dict[str, Any]:
+    selected_project = _selected_project(project_id_filter, project_id)
+    return db.usage_summary(project_id=selected_project)