agentloop-profiler 0.4.0__py3-none-any.whl

agentloop/replay.py

@@ -0,0 +1,424 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from agentloop.markdown import markdown_code_span, markdown_table_cell
+
+
+@dataclass(frozen=True)
+class ReplayGates:
+    max_cost_regression_pct: float = 0.0
+    max_latency_regression_pct: float = 0.0
+    min_latency_improvement_pct: float = 0.0
+    min_cost_improvement_pct: float = 0.0
+    require_retry_non_increase: bool = True
+    require_schema_valid: bool = False
+    min_quality_score: float | None = None
+
+
+def build_replay_report(
+    baseline_trace: Any,
+    candidate_trace: Any,
+    *,
+    gates: ReplayGates | None = None,
+    quality_report: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    gates = gates or ReplayGates()
+    baseline_report = baseline_trace.report()
+    candidate_report = candidate_trace.report()
+    baseline = _trace_summary(baseline_trace, baseline_report)
+    candidate = _trace_summary(candidate_trace, candidate_report)
+    if quality_report is not None:
+        baseline["quality_score"] = float(quality_report.get("baseline_score", 0.0))
+        candidate["quality_score"] = float(quality_report.get("candidate_score", 0.0))
+    deltas = _deltas(baseline, candidate)
+    gate_results = _gate_results(deltas, baseline, candidate, gates, quality_report=quality_report)
+    passed = all(item["passed"] for item in gate_results)
+
+    return {
+        "baseline": baseline,
+        "candidate": candidate,
+        "deltas": deltas,
+        "gates": {
+            "passed": passed,
+            "config": {
+                "max_cost_regression_pct": gates.max_cost_regression_pct,
+                "max_latency_regression_pct": gates.max_latency_regression_pct,
+                "min_latency_improvement_pct": gates.min_latency_improvement_pct,
+                "min_cost_improvement_pct": gates.min_cost_improvement_pct,
+                "require_retry_non_increase": gates.require_retry_non_increase,
+                "require_schema_valid": gates.require_schema_valid,
+                "min_quality_score": gates.min_quality_score,
+            },
+            "results": gate_results,
+        },
+        "quality": quality_report,
+        "summary": _summary(deltas, passed),
+    }
+
+
+def replay_report_to_markdown(report: dict[str, Any]) -> str:
+    baseline = report["baseline"]
+    candidate = report["candidate"]
+    deltas = report["deltas"]
+    gates = report["gates"]
+    status = "passed" if gates["passed"] else "failed"
+
+    lines = [
+        "# AgentLoop Replay Report",
+        "",
+        f"- Status: {status}",
+        f"- Baseline: {markdown_code_span(baseline['name'])} "
+        f"{markdown_code_span(baseline['run_id'])}",
+        f"- Candidate: {markdown_code_span(candidate['name'])} "
+        f"{markdown_code_span(candidate['run_id'])}",
+        "",
+        "## Metric Deltas",
+        "",
+        "| Metric | Baseline | Candidate | Delta | Improvement |",
+        "|---|---:|---:|---:|---:|",
+        _metric_row(
+            "Runtime",
+            baseline["runtime_ms"],
+            candidate["runtime_ms"],
+            deltas["runtime_ms_delta"],
+            deltas["latency_improvement_pct"],
+            "ms",
+        ),
+        _metric_row(
+            "Cost",
+            baseline["estimated_cost_usd"],
+            candidate["estimated_cost_usd"],
+            deltas["cost_usd_delta"],
+            deltas["cost_improvement_pct"],
+            "usd",
+        ),
+        _metric_row(
+            "Input tokens",
+            baseline["input_tokens"],
+            candidate["input_tokens"],
+            deltas["input_tokens_delta"],
+            deltas["input_token_improvement_pct"],
+            "count",
+        ),
+        _metric_row(
+            "Output tokens",
+            baseline["output_tokens"],
+            candidate["output_tokens"],
+            deltas["output_tokens_delta"],
+            deltas["output_token_improvement_pct"],
+            "count",
+        ),
+        _metric_row(
+            "Retries",
+            baseline["retry_count"],
+            candidate["retry_count"],
+            deltas["retry_count_delta"],
+            deltas["retry_improvement_pct"],
+            "count",
+        ),
+        _metric_row(
+            "Tool calls",
+            baseline["tool_call_count"],
+            candidate["tool_call_count"],
+            deltas["tool_call_count_delta"],
+            deltas["tool_call_improvement_pct"],
+            "count",
+        ),
+        _metric_row(
+            "Model calls",
+            baseline["model_call_count"],
+            candidate["model_call_count"],
+            deltas["model_call_count_delta"],
+            deltas["model_call_improvement_pct"],
+            "count",
+        ),
+    ]
+    if (
+        baseline.get("schema_validity_pct") is not None
+        or candidate.get("schema_validity_pct") is not None
+    ):
+        lines.append(
+            _metric_row(
+                "Schema validity",
+                baseline.get("schema_validity_pct") or 0.0,
+                candidate.get("schema_validity_pct") or 0.0,
+                deltas["schema_validity_delta_pct"],
+                deltas["schema_validity_improvement_pct"],
+                "pct",
+            )
+        )
+    if baseline.get("quality_score") is not None or candidate.get("quality_score") is not None:
+        lines.append(
+            _metric_row(
+                "Quality score",
+                baseline.get("quality_score") or 0.0,
+                candidate.get("quality_score") or 0.0,
+                deltas["quality_score_delta"],
+                deltas["quality_score_improvement_pct"],
+                "score",
+            )
+        )
+    if report.get("quality"):
+        quality = report["quality"]
+        lines.extend(
+            [
+                "",
+                "## Quality",
+                "",
+                f"- Cases: {quality['case_count']}",
+                f"- Candidate score: {quality['candidate_score']:.4f}",
+                f"- Quality delta: {quality['quality_delta']:.4f}",
+                f"- Failed cases: {quality['failed_case_count']}",
+            ]
+        )
+    lines.extend(
+        [
+            "",
+            "## Gates",
+            "",
+            "| Gate | Status | Detail |",
+            "|---|---|---|",
+        ]
+    )
+    for item in gates["results"]:
+        gate_status = "pass" if item["passed"] else "fail"
+        lines.append(
+            f"| {markdown_table_cell(item['name'])} | {gate_status} | "
+            f"{markdown_table_cell(item['detail'])} |"
+        )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _trace_summary(trace: Any, report: dict[str, Any]) -> dict[str, Any]:
+    metadata = getattr(trace, "metadata", {}) or {}
+    return {
+        "run_id": trace.run_id,
+        "name": trace.name,
+        "runtime_ms": float(report.get("total_runtime_ms", 0.0) or 0.0),
+        "estimated_cost_usd": float(report.get("estimated_cost_usd", 0.0) or 0.0),
+        "input_tokens": int(report.get("input_tokens", 0) or 0),
+        "output_tokens": int(report.get("output_tokens", 0) or 0),
+        "retry_count": int(report.get("retry_count", 0) or 0),
+        "tool_call_count": int(report.get("tool_call_count", 0) or 0),
+        "model_call_count": int(report.get("model_call_count", 0) or 0),
+        "schema_validity_pct": _optional_float(
+            report.get("schema_validity_pct")
+            if report.get("schema_validity_pct") is not None
+            else metadata.get("schema_validity_pct")
+        ),
+        "schema_valid": _optional_bool(
+            report.get("schema_valid")
+            if report.get("schema_valid") is not None
+            else metadata.get("schema_valid")
+        ),
+        "quality_score": _optional_float(
+            report.get("quality_score")
+            if report.get("quality_score") is not None
+            else metadata.get("quality_score")
+        ),
+    }
+
+
+def _deltas(baseline: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]:
+    base_runtime = float(baseline.get("runtime_ms", baseline.get("total_runtime_ms", 0.0)) or 0.0)
+    cand_runtime = float(candidate.get("runtime_ms", candidate.get("total_runtime_ms", 0.0)) or 0.0)
+    base_cost = float(baseline.get("estimated_cost_usd", 0.0) or 0.0)
+    cand_cost = float(candidate.get("estimated_cost_usd", 0.0) or 0.0)
+    base_input = int(baseline.get("input_tokens", 0) or 0)
+    cand_input = int(candidate.get("input_tokens", 0) or 0)
+    base_output = int(baseline.get("output_tokens", 0) or 0)
+    cand_output = int(candidate.get("output_tokens", 0) or 0)
+    base_retries = int(baseline.get("retry_count", 0) or 0)
+    cand_retries = int(candidate.get("retry_count", 0) or 0)
+    base_tools = int(baseline.get("tool_call_count", 0) or 0)
+    cand_tools = int(candidate.get("tool_call_count", 0) or 0)
+    base_models = int(baseline.get("model_call_count", 0) or 0)
+    cand_models = int(candidate.get("model_call_count", 0) or 0)
+    base_schema = _optional_float(baseline.get("schema_validity_pct"))
+    if base_schema is None:
+        base_schema = _schema_validity_from_bool(baseline.get("schema_valid"))
+    cand_schema = _optional_float(candidate.get("schema_validity_pct"))
+    if cand_schema is None:
+        cand_schema = _schema_validity_from_bool(candidate.get("schema_valid"))
+    base_quality = _optional_float(baseline.get("quality_score"))
+    cand_quality = _optional_float(candidate.get("quality_score"))
+
+    return {
+        "runtime_ms_delta": round(cand_runtime - base_runtime, 3),
+        "latency_improvement_pct": _improvement_pct(base_runtime, cand_runtime),
+        "latency_regression_pct": _regression_pct(base_runtime, cand_runtime),
+        "cost_usd_delta": round(cand_cost - base_cost, 6),
+        "cost_improvement_pct": _improvement_pct(base_cost, cand_cost),
+        "cost_regression_pct": _regression_pct(base_cost, cand_cost),
+        "input_tokens_delta": cand_input - base_input,
+        "input_token_improvement_pct": _improvement_pct(base_input, cand_input),
+        "output_tokens_delta": cand_output - base_output,
+        "output_token_improvement_pct": _improvement_pct(base_output, cand_output),
+        "retry_count_delta": cand_retries - base_retries,
+        "retry_improvement_pct": _improvement_pct(base_retries, cand_retries),
+        "tool_call_count_delta": cand_tools - base_tools,
+        "tool_call_improvement_pct": _improvement_pct(base_tools, cand_tools),
+        "model_call_count_delta": cand_models - base_models,
+        "model_call_improvement_pct": _improvement_pct(base_models, cand_models),
+        "schema_validity_delta_pct": round((cand_schema or 0.0) - (base_schema or 0.0), 3),
+        "schema_validity_improvement_pct": _relative_gain_pct(base_schema, cand_schema),
+        "quality_score_delta": round((cand_quality or 0.0) - (base_quality or 0.0), 4),
+        "quality_score_improvement_pct": _relative_gain_pct(base_quality, cand_quality),
+    }
+
+
+def _gate_results(
+    deltas: dict[str, Any],
+    baseline: dict[str, Any],
+    candidate: dict[str, Any],
+    gates: ReplayGates,
+    *,
+    quality_report: dict[str, Any] | None = None,
+) -> list[dict[str, Any]]:
+    results = [
+        _gate(
+            "latency_regression",
+            deltas["latency_regression_pct"] <= gates.max_latency_regression_pct,
+            f"{deltas['latency_regression_pct']:.2f}% regression <= {gates.max_latency_regression_pct:.2f}% allowed",
+        ),
+        _gate(
+            "cost_regression",
+            deltas["cost_regression_pct"] <= gates.max_cost_regression_pct,
+            f"{deltas['cost_regression_pct']:.2f}% regression <= {gates.max_cost_regression_pct:.2f}% allowed",
+        ),
+        _gate(
+            "latency_improvement",
+            deltas["latency_improvement_pct"] >= gates.min_latency_improvement_pct,
+            f"{deltas['latency_improvement_pct']:.2f}% improvement >= {gates.min_latency_improvement_pct:.2f}% required",
+        ),
+        _gate(
+            "cost_improvement",
+            deltas["cost_improvement_pct"] >= gates.min_cost_improvement_pct,
+            f"{deltas['cost_improvement_pct']:.2f}% improvement >= {gates.min_cost_improvement_pct:.2f}% required",
+        ),
+    ]
+    if gates.require_retry_non_increase:
+        base_retries = int(baseline.get("retry_count", 0) or 0)
+        cand_retries = int(candidate.get("retry_count", 0) or 0)
+        results.append(
+            _gate(
+                "retry_non_increase",
+                cand_retries <= base_retries,
+                f"{cand_retries} candidate retries <= {base_retries} baseline retries",
+            )
+        )
+    if gates.require_schema_valid:
+        schema_valid = _candidate_schema_valid(candidate)
+        results.append(
+            _gate(
+                "schema_valid",
+                schema_valid,
+                "candidate schema validity passed"
+                if schema_valid
+                else "candidate schema validity missing or failed",
+            )
+        )
+    if gates.min_quality_score is not None:
+        quality_score = _optional_float(candidate.get("quality_score"))
+        fixture_passed = quality_report is None or bool(quality_report.get("passed"))
+        passed = (
+            quality_score is not None
+            and quality_score >= gates.min_quality_score
+            and fixture_passed
+        )
+        detail = (
+            f"{quality_score:.4f} candidate quality >= {gates.min_quality_score:.4f} required"
+            if quality_score is not None
+            else f"candidate quality score missing; {gates.min_quality_score:.4f} required"
+        )
+        if quality_report is not None and not fixture_passed:
+            detail += f"; {quality_report.get('failed_case_count', 0)} fixture case(s) failed"
+        results.append(_gate("quality_score", passed, detail))
+    return results
+
+
+def _gate(name: str, passed: bool, detail: str) -> dict[str, Any]:
+    return {"name": name, "passed": passed, "detail": detail}
+
+
+def _summary(deltas: dict[str, Any], passed: bool) -> str:
+    status = "passed" if passed else "failed"
+    return (
+        f"Replay {status}: latency improvement {deltas['latency_improvement_pct']:.2f}%, "
+        f"cost improvement {deltas['cost_improvement_pct']:.2f}%, "
+        f"retry delta {deltas['retry_count_delta']}."
+    )
+
+
+def _improvement_pct(baseline: float, candidate: float) -> float:
+    if baseline == 0:
+        return 0.0 if candidate == 0 else -100.0
+    return round(((baseline - candidate) / baseline) * 100, 2)
+
+
+def _relative_gain_pct(baseline: float | None, candidate: float | None) -> float:
+    if baseline is None:
+        return 0.0 if candidate is None else 100.0
+    if baseline == 0:
+        return 0.0 if not candidate else 100.0
+    return round((((candidate or 0.0) - baseline) / baseline) * 100, 2)
+
+
+def _regression_pct(baseline: float, candidate: float) -> float:
+    if baseline == 0:
+        return 0.0 if candidate == 0 else 100.0
+    return round(max(0.0, ((candidate - baseline) / baseline) * 100), 2)
+
+
+def _metric_row(
+    metric: str, baseline: float, candidate: float, delta: float, improvement: float, kind: str
+) -> str:
+    return (
+        f"| {metric} | {_format_metric(baseline, kind)} | {_format_metric(candidate, kind)} | "
+        f"{_format_metric(delta, kind)} | {improvement:.2f}% |"
+    )
+
+
+def _format_metric(value: float, kind: str) -> str:
+    if kind == "usd":
+        return f"${value:.6f}"
+    if kind == "ms":
+        return f"{value:.3f}ms"
+    if kind == "pct":
+        return f"{value:.3f}%"
+    if kind == "score":
+        return f"{value:.4f}"
+    return str(int(value))
+
+
+def _optional_float(value: Any) -> float | None:
+    if value is None or value == "":
+        return None
+    return float(value)
+
+
+def _optional_bool(value: Any) -> bool | None:
+    if value is None or value == "":
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"1", "true", "yes", "pass", "passed", "valid"}
+    return bool(value)
+
+
+def _schema_validity_from_bool(value: Any) -> float | None:
+    parsed = _optional_bool(value)
+    if parsed is None:
+        return None
+    return 100.0 if parsed else 0.0
+
+
+def _candidate_schema_valid(candidate: dict[str, Any]) -> bool:
+    schema_valid = _optional_bool(candidate.get("schema_valid"))
+    if schema_valid is not None:
+        return schema_valid
+    validity_pct = _optional_float(candidate.get("schema_validity_pct"))
+    return validity_pct is not None and validity_pct >= 100.0

agentloop/runtime.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+_TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+class _ClearSentinel:
+    """Sentinel passed to :func:`init` to explicitly clear an optional value.
+
+    ``init(api_key=None)`` means "keep the current value" (so environment and
+    prior configuration survive a reconfigure). Passing ``CLEAR`` instead sets
+    the value back to ``None``.
+    """
+
+    def __repr__(self) -> str:
+        return "CLEAR"
+
+
+CLEAR = _ClearSentinel()
+
+
+class FinalizationError(RuntimeError):
+    """Raised by :func:`finalize_trace` when a destination fails and
+    ``fail_silently`` is ``False``.
+
+    ``result`` holds the partial finalization result for destinations that
+    completed before the failure, so already-completed work is not lost.
+    ``errors`` lists the destination-tagged failures.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        result: dict[str, Any],
+        errors: list[dict[str, str]],
+    ) -> None:
+        super().__init__(message)
+        self.result = result
+        self.errors = errors
+
+
+@dataclass
+class AgentLoopRuntimeConfig:
+    api_url: str = "http://127.0.0.1:8000"
+    api_key: str | None = None
+    project_id: str = "default"
+    auto_upload: bool = False
+    auto_store: bool = False
+    fail_silently: bool = True
+    export_dir: Path | None = None
+
+
+_runtime = AgentLoopRuntimeConfig()
+_last_error: str | None = None
+
+
+def _env_bool(name: str, default: bool = False) -> bool:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return value.lower() in _TRUE_VALUES
+
+
+def init(
+    *,
+    api_url: str | None = None,
+    api_key: str | _ClearSentinel | None = None,
+    project_id: str | None = None,
+    auto_upload: bool | None = None,
+    auto_store: bool | None = None,
+    fail_silently: bool | None = None,
+    export_dir: str | Path | _ClearSentinel | None = None,
+) -> AgentLoopRuntimeConfig:
+    """Configure AgentLoop once at process startup.
+
+    Examples:
+
+    ```python
+    import agentloop
+
+    agentloop.init(api_key="al_xxx", project_id="demo", auto_upload=True)
+    ```
+
+    Environment variables are also supported:
+    - `AGENTLOOP_API_URL`
+    - `AGENTLOOP_API_KEY`
+    - `AGENTLOOP_PROJECT_ID`
+    - `AGENTLOOP_AUTO_UPLOAD`
+    - `AGENTLOOP_AUTO_STORE`
+    - `AGENTLOOP_FAIL_SILENTLY`
+    - `AGENTLOOP_EXPORT_DIR`
+
+    Optional values (`api_key`, `export_dir`) treat ``None`` as "keep the current
+    value". Pass :data:`CLEAR` to reset one of them back to ``None``.
+    """
+
+    global _runtime
+
+    if api_key is CLEAR:
+        resolved_api_key: str | None = None
+    elif api_key is not None:
+        resolved_api_key = api_key  # type: ignore[assignment]
+    else:
+        resolved_api_key = os.getenv("AGENTLOOP_API_KEY", _runtime.api_key)
+
+    if export_dir is CLEAR:
+        resolved_export_dir: Path | None = None
+    elif export_dir is not None:
+        resolved_export_dir = Path(export_dir)  # type: ignore[arg-type]
+    else:
+        env_export_dir = os.getenv("AGENTLOOP_EXPORT_DIR")
+        resolved_export_dir = Path(env_export_dir) if env_export_dir else _runtime.export_dir
+
+    _runtime = AgentLoopRuntimeConfig(
+        api_url=api_url or os.getenv("AGENTLOOP_API_URL", _runtime.api_url),
+        api_key=resolved_api_key,
+        project_id=project_id or os.getenv("AGENTLOOP_PROJECT_ID", _runtime.project_id),
+        auto_upload=auto_upload
+        if auto_upload is not None
+        else _env_bool("AGENTLOOP_AUTO_UPLOAD", _runtime.auto_upload),
+        auto_store=auto_store
+        if auto_store is not None
+        else _env_bool("AGENTLOOP_AUTO_STORE", _runtime.auto_store),
+        fail_silently=fail_silently
+        if fail_silently is not None
+        else _env_bool("AGENTLOOP_FAIL_SILENTLY", _runtime.fail_silently),
+        export_dir=resolved_export_dir,
+    )
+    return _runtime
+
+
+def configure_from_env() -> AgentLoopRuntimeConfig:
+    return init()
+
+
+def get_runtime_config() -> AgentLoopRuntimeConfig:
+    return _runtime
+
+
+def get_last_error() -> str | None:
+    return _last_error
+
+
+def reset_runtime() -> None:
+    global _runtime, _last_error
+    _runtime = AgentLoopRuntimeConfig()
+    _last_error = None
+
+
+def should_auto_export() -> bool:
+    return bool(_runtime.auto_upload or _runtime.auto_store or _runtime.export_dir)
+
+
+def finalize_trace(trace: Any) -> dict[str, Any]:
+    """Apply configured end-of-run side effects.
+
+    Depending on runtime config, this exports a JSON file, saves to the local
+    persistent store, and/or uploads to the configured API. Each destination runs
+    in its own error boundary in a deterministic order (export, store, upload), so
+    a failure in one cannot prevent the others from running.
+
+    When ``fail_silently`` is ``True`` (the default), every configured destination
+    is attempted and failures are collected in ``result["errors"]``, each tagged
+    with its ``destination``. When ``fail_silently`` is ``False``, the first failing
+    destination raises :class:`FinalizationError` (fail-fast); its ``result``
+    attribute still carries the destinations that completed before the failure.
+
+    A finalization in which every attempted destination succeeds clears any stale
+    process-global error so :func:`get_last_error` reflects the latest outcome.
+    """
+
+    global _last_error
+    result: dict[str, Any] = {
+        "exported_path": None,
+        "stored": False,
+        "uploaded": False,
+        "errors": [],
+    }
+
+    def _do_export() -> None:
+        export_dir = _runtime.export_dir
+        if export_dir is None:  # pragma: no cover - guarded before registration
+            return
+        export_dir.mkdir(parents=True, exist_ok=True)
+        out = export_dir / f"{trace.run_id}.json"
+        trace.export_json(out)
+        result["exported_path"] = str(out)
+
+    def _do_store() -> None:
+        from agentloop.store import get_store
+
+        db = get_store()
+        db.init()
+        db.save_trace(trace, project_id=_runtime.project_id)
+        result["stored"] = True
+
+    def _do_upload() -> None:
+        from agentloop.client import AgentLoopClient
+
+        client = AgentLoopClient(base_url=_runtime.api_url, api_key=_runtime.api_key)
+        result["upload_response"] = client.upload_trace(trace)
+        result["uploaded"] = True
+
+    destinations: list[tuple[str, Any]] = []
+    if _runtime.export_dir is not None:
+        destinations.append(("export", _do_export))
+    if _runtime.auto_store:
+        destinations.append(("store", _do_store))
+    if _runtime.auto_upload:
+        destinations.append(("upload", _do_upload))
+
+    errors: list[dict[str, str]] = []
+    for destination, action in destinations:
+        try:
+            action()
+        except Exception as exc:
+            errors.append({"destination": destination, "error": str(exc)})
+            if not _runtime.fail_silently:
+                result["errors"] = errors
+                _last_error = f"{destination}: {exc}"
+                raise FinalizationError(
+                    f"{destination} finalization failed: {exc}",
+                    result=result,
+                    errors=errors,
+                ) from exc
+
+    result["errors"] = errors
+    if errors:
+        last = errors[-1]
+        _last_error = f"{last['destination']}: {last['error']}"
+    elif destinations:
+        # Every attempted destination succeeded; drop any stale error state.
+        _last_error = None
+
+    return result
+
+
+# Load env-driven defaults on import without forcing users to call init().
+configure_from_env()