agentloop-profiler 0.4.0__py3-none-any.whl

agentloop/client.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from agentloop.tracer import AgentTrace
+
+
+class AgentLoopClientError(RuntimeError):
+    pass
+
+
+@dataclass
+class AgentLoopClient:
+    """Tiny dependency-free client for sending traces to an AgentLoop API server."""
+
+    base_url: str = "http://127.0.0.1:8000"
+    api_key: str | None = None
+    admin_api_key: str | None = None
+    timeout_s: float = 10.0
+
+    @classmethod
+    def from_env(cls) -> "AgentLoopClient":
+        return cls(
+            base_url=os.getenv("AGENTLOOP_API_URL", "http://127.0.0.1:8000"),
+            api_key=os.getenv("AGENTLOOP_API_KEY"),
+            admin_api_key=os.getenv("AGENTLOOP_ADMIN_API_KEY"),
+            timeout_s=float(os.getenv("AGENTLOOP_TIMEOUT_S", "10")),
+        )
+
+    def health(self) -> dict[str, Any]:
+        return self._request("GET", "/health")
+
+    def create_api_key(self, project_id: str = "default", name: str = "default") -> dict[str, Any]:
+        return self._request("POST", "/api-keys", {"project_id": project_id, "name": name})
+
+    def upload_trace(self, trace: AgentTrace | str | Path | dict[str, Any]) -> dict[str, Any]:
+        if isinstance(trace, AgentTrace):
+            payload = trace.to_dict()
+        elif isinstance(trace, dict):
+            payload = trace
+        else:
+            payload = json.loads(Path(trace).read_text(encoding="utf-8"))
+        return self._request("POST", "/traces", payload)
+
+    def list_traces(self, project_id: str | None = None) -> dict[str, Any]:
+        route = "/traces"
+        if project_id:
+            route += "?" + urllib.parse.urlencode({"project_id": project_id})
+        return self._request("GET", route)
+
+    def usage_summary(self, project_id: str | None = None) -> dict[str, Any]:
+        route = "/usage"
+        if project_id:
+            route += "?" + urllib.parse.urlencode({"project_id": project_id})
+        return self._request("GET", route)
+
+    def get_report(self, run_id: str) -> dict[str, Any]:
+        encoded_run_id = urllib.parse.quote(run_id, safe="")
+        return self._request("GET", f"/traces/{encoded_run_id}/report")
+
+    def get_optimization_plan(self, run_id: str) -> dict[str, Any]:
+        encoded_run_id = urllib.parse.quote(run_id, safe="")
+        return self._request("GET", f"/traces/{encoded_run_id}/optimize")
+
+    def get_diagnosis(self, run_id: str) -> dict[str, Any]:
+        encoded_run_id = urllib.parse.quote(run_id, safe="")
+        return self._request("GET", f"/traces/{encoded_run_id}/diagnose")
+
+    def list_findings(
+        self, project_id: str | None = None, status: str | None = None
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {}
+        if project_id:
+            params["project_id"] = project_id
+        if status:
+            params["status"] = status
+        route = "/findings"
+        if params:
+            route += "?" + urllib.parse.urlencode(params)
+        return self._request("GET", route)
+
+    def optimization_queue(self, project_id: str | None = None) -> dict[str, Any]:
+        route = "/optimization-queue"
+        if project_id:
+            route += "?" + urllib.parse.urlencode({"project_id": project_id})
+        return self._request("GET", route)
+
+    def github_issue_drafts(self, project_id: str | None = None, limit: int = 5) -> dict[str, Any]:
+        params: dict[str, Any] = {"limit": limit}
+        if project_id:
+            params["project_id"] = project_id
+        return self._request(
+            "GET", "/optimization-queue/github-issues?" + urllib.parse.urlencode(params)
+        )
+
+    def build_quality_report(
+        self,
+        fixtures: list[dict[str, Any]],
+        *,
+        baseline_run_id: str | None = None,
+        candidate_run_id: str | None = None,
+        min_score: float | None = None,
+    ) -> dict[str, Any]:
+        return self._request(
+            "POST",
+            "/quality-report",
+            {
+                "fixtures": fixtures,
+                "baseline_run_id": baseline_run_id,
+                "candidate_run_id": candidate_run_id,
+                "min_score": min_score,
+            },
+        )
+
+    def get_value_report(
+        self,
+        run_id: str,
+        *,
+        runs_per_month: int | None = None,
+        engineer_hourly_rate_usd: float | None = None,
+        incident_cost_usd: float | None = None,
+    ) -> dict[str, Any]:
+        params: dict[str, Any] = {}
+        if runs_per_month is not None:
+            params["runs_per_month"] = runs_per_month
+        if engineer_hourly_rate_usd is not None:
+            params["engineer_hourly_rate_usd"] = engineer_hourly_rate_usd
+        if incident_cost_usd is not None:
+            params["incident_cost_usd"] = incident_cost_usd
+        encoded_run_id = urllib.parse.quote(run_id, safe="")
+        route = f"/traces/{encoded_run_id}/value"
+        if params:
+            route += "?" + urllib.parse.urlencode(params)
+        return self._request("GET", route)
+
+    def _request(
+        self, method: str, path: str, payload: dict[str, Any] | None = None
+    ) -> dict[str, Any]:
+        parsed_base_url = urllib.parse.urlsplit(self.base_url)
+        if parsed_base_url.scheme not in {"http", "https"} or not parsed_base_url.netloc:
+            raise AgentLoopClientError("AgentLoop API URL must use http:// or https://")
+        url = self.base_url.rstrip("/") + path
+        body = None if payload is None else json.dumps(payload).encode("utf-8")
+        headers = {"Accept": "application/json"}
+        if payload is not None:
+            headers["Content-Type"] = "application/json"
+        if self.api_key:
+            headers["X-AgentLoop-Key"] = self.api_key
+        if self.admin_api_key:
+            headers["X-AgentLoop-Admin-Key"] = self.admin_api_key
+        req = urllib.request.Request(url=url, data=body, method=method, headers=headers)
+        try:
+            with urllib.request.urlopen(  # nosec B310 - base URL scheme is validated above.
+                req, timeout=self.timeout_s
+            ) as response:
+                return json.loads(response.read().decode("utf-8"))
+        except urllib.error.HTTPError as exc:
+            detail = exc.read().decode("utf-8", errors="ignore")
+            raise AgentLoopClientError(f"AgentLoop API returned {exc.code}: {detail}") from exc
+        except urllib.error.URLError as exc:
+            raise AgentLoopClientError(
+                f"Could not reach AgentLoop API at {url}: {exc.reason}"
+            ) from exc

agentloop/config.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import os
+
+_POSTGRES_URL_ENV_VARS = ("AGENTLOOP_DATABASE_URL", "DATABASE_URL")
+_POSTGRES_LIBPQ_SOURCE_ENV_VARS = ("PGHOST", "PGHOSTADDR", "PGSERVICE")
+
+
+def get_api_key() -> str | None:
+    return os.getenv("AGENTLOOP_API_KEY")
+
+
+def get_admin_api_key() -> str | None:
+    return os.getenv("AGENTLOOP_ADMIN_API_KEY")
+
+
+def require_api_key() -> bool:
+    return os.getenv("AGENTLOOP_REQUIRE_API_KEY", "false").lower() in {"1", "true", "yes", "on"}
+
+
+def get_api_url() -> str:
+    return os.getenv("AGENTLOOP_API_URL", "http://127.0.0.1:8000")
+
+
+def get_cors_origins() -> list[str]:
+    value = os.getenv("AGENTLOOP_CORS_ORIGINS", "")
+    return [origin.strip() for origin in value.split(",") if origin.strip()]
+
+
+def get_postgres_dsn() -> str | None:
+    """Return the explicit Postgres DSN, preserving AgentLoop override precedence."""
+    for name in _POSTGRES_URL_ENV_VARS:
+        value = os.getenv(name)
+        if value:
+            return value
+    return None
+
+
+def get_postgres_password_file() -> str | None:
+    """Return the optional file containing a libpq password."""
+    return os.getenv("AGENTLOOP_POSTGRES_PASSWORD_FILE") or None
+
+
+def postgres_connection_source() -> str | None:
+    """Describe the configured Postgres connection source without exposing secrets."""
+    if get_postgres_dsn():
+        return "database URL"
+    if any(os.getenv(name) for name in _POSTGRES_LIBPQ_SOURCE_ENV_VARS):
+        return "libpq environment"
+    return None

agentloop/costs.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+MODEL_PRICES = {
+    "gpt-4.1": {"input": 2.00, "output": 8.00},
+    "gpt-4.1-mini": {"input": 0.40, "output": 1.60},
+    "gpt-4o-mini": {"input": 0.15, "output": 0.60},
+    "default": {"input": 1.00, "output": 3.00},
+}
+
+
+def estimate_cost_usd(model: str | None, input_tokens: int, output_tokens: int) -> float:
+    price = MODEL_PRICES.get(model or "default", MODEL_PRICES["default"])
+    input_cost = (input_tokens / 1000000) * price["input"]
+    output_cost = (output_tokens / 1000000) * price["output"]
+    return round(input_cost + output_cost, 6)

agentloop/decorators.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import inspect
+from functools import wraps
+from typing import Any, Callable, TypeVar, overload
+
+from agentloop.tracer import current_trace, trace_agent, trace_model_call, trace_tool_call
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def _call_metadata(args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]:
+    metadata: dict[str, Any] = {"integration": "agentloop.decorator"}
+    if args:
+        metadata["arg_count"] = len(args)
+    if kwargs:
+        metadata["kwarg_keys"] = sorted(str(key) for key in kwargs.keys())
+    return metadata
+
+
+@overload
+def traceable(fn: F) -> F: ...
+
+
+@overload
+def traceable(
+    fn: None = None,
+    *,
+    name: str | None = None,
+    kind: str = "tool",
+    model: str | None = None,
+    root: bool = False,
+    agent_name: str | None = None,
+) -> Callable[[F], F]: ...
+
+
+def traceable(
+    fn: F | None = None,
+    *,
+    name: str | None = None,
+    kind: str = "tool",
+    model: str | None = None,
+    root: bool = False,
+    agent_name: str | None = None,
+) -> F | Callable[[F], F]:
+    """Trace a normal Python function without changing its implementation.
+
+    Examples:
+
+    ```python
+    import agentloop
+
+
+    @agentloop.traceable(root=True, agent_name="research_agent")
+    def run_agent(question: str):
+        return plan(question)
+
+
+    @agentloop.traceable(kind="model", model="gpt-4.1-mini")
+    def plan(question: str):
+        return client.responses.create(...)
+
+
+    @agentloop.traceable(kind="tool")
+    def search_web(query: str):
+        return search(query)
+    ```
+
+    `root=True` starts an AgentLoop run when no active run exists. Nested calls are
+    recorded as spans inside that run. This gives teams a low-friction migration
+    path when they do not use LangGraph, CrewAI, or another supported framework.
+    """
+
+    normalized_kind = kind.lower().replace("-", "_")
+    if normalized_kind not in {"tool", "model"}:
+        raise ValueError("kind must be 'tool' or 'model'")
+
+    def decorator(func: F) -> F:
+        span_name = name or getattr(
+            func, "__name__", getattr(func, "__qualname__", "agentloop.call")
+        )
+        run_name = agent_name or span_name
+
+        if inspect.iscoroutinefunction(func):
+
+            @wraps(func)
+            async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+                if root and current_trace() is None:
+                    with trace_agent(run_name, metadata={"entrypoint": span_name}):
+                        return await _run_async_span(
+                            func, span_name, normalized_kind, model, args, kwargs
+                        )
+                return await _run_async_span(func, span_name, normalized_kind, model, args, kwargs)
+
+            return async_wrapper  # type: ignore[return-value]
+
+        @wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            if root and current_trace() is None:
+                with trace_agent(run_name, metadata={"entrypoint": span_name}):
+                    return _run_sync_span(func, span_name, normalized_kind, model, args, kwargs)
+            return _run_sync_span(func, span_name, normalized_kind, model, args, kwargs)
+
+        return wrapper  # type: ignore[return-value]
+
+    if fn is not None:
+        return decorator(fn)
+    return decorator
+
+
+async def _run_async_span(
+    func: Callable[..., Any],
+    span_name: str,
+    kind: str,
+    model: str | None,
+    args: tuple[Any, ...],
+    kwargs: dict[str, Any],
+) -> Any:
+    metadata = _call_metadata(args, kwargs)
+    if kind == "model":
+        with trace_model_call(span_name, model=model, metadata=metadata):
+            return await func(*args, **kwargs)
+    with trace_tool_call(span_name, metadata=metadata):
+        return await func(*args, **kwargs)
+
+
+def _run_sync_span(
+    func: Callable[..., Any],
+    span_name: str,
+    kind: str,
+    model: str | None,
+    args: tuple[Any, ...],
+    kwargs: dict[str, Any],
+) -> Any:
+    metadata = _call_metadata(args, kwargs)
+    if kind == "model":
+        with trace_model_call(span_name, model=model, metadata=metadata):
+            return func(*args, **kwargs)
+    with trace_tool_call(span_name, metadata=metadata):
+        return func(*args, **kwargs)
+
+
+def trace_model(
+    *,
+    name: str | None = None,
+    model: str | None = None,
+    root: bool = False,
+    agent_name: str | None = None,
+) -> Callable[[F], F]:
+    """Convenience decorator for model-call-like functions."""
+
+    return traceable(name=name, kind="model", model=model, root=root, agent_name=agent_name)
+
+
+def trace_tool(
+    *, name: str | None = None, root: bool = False, agent_name: str | None = None
+) -> Callable[[F], F]:
+    """Convenience decorator for tool-call-like functions."""
+
+    return traceable(name=name, kind="tool", root=root, agent_name=agent_name)

agentloop/demo.py

@@ -0,0 +1,283 @@
+from __future__ import annotations
+
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from agentloop import trace_agent, trace_model_call, trace_retry, trace_tool_call
+
+SYSTEM_LONG = (
+    "You are a careful research agent. Use evidence, verify claims, and write concise reports. "
+    * 20
+)
+SYSTEM_SHORT = (
+    "You are a careful research agent. Use evidence, verify claims, and write concise reports. " * 4
+)
+
+
+def _demo_metadata(**values: object) -> dict[str, object]:
+    metadata: dict[str, object] = {"synthetic": True, "source": "agentloop_demo"}
+    metadata.update(values)
+    return metadata
+
+
+def sleep_ms(ms: int) -> None:
+    time.sleep(ms / 1000)
+
+
+def fake_search(query: str) -> list[str]:
+    sleep_ms(300)
+    return [f"source-{i}-{query}" for i in range(1, 6)]
+
+
+def fake_read(source: str) -> str:
+    sleep_ms(250)
+    return f"Content from {source}. " * 30
+
+
+def inspect_result(result: str) -> str:
+    sleep_ms(120)
+    return f"inspected {result}"
+
+
+def run_baseline(out_dir: str | Path = "runs") -> Path:
+    out_path = Path(out_dir) / "research_agent_baseline.json"
+    with trace_agent("research_agent_baseline", metadata=_demo_metadata()) as trace:
+        with trace_model_call(
+            "plan",
+            model="gpt-4.1",
+            input_text=SYSTEM_LONG + "Plan research on agent inference runtimes.",
+            output_tokens=220,
+        ):
+            sleep_ms(400)
+
+        with trace_tool_call("search_web", metadata={"query": "agent inference runtime"}):
+            sources = fake_search("agent inference runtime")
+
+        summaries = []
+        for source in sources:
+            with trace_tool_call("read_source", metadata={"source": source}):
+                content = fake_read(source)
+            with trace_model_call(
+                "summarize_source",
+                model="gpt-4.1",
+                input_text=SYSTEM_LONG + content,
+                output_tokens=350,
+            ):
+                sleep_ms(350)
+                summaries.append("summary for " + source)
+
+        with trace_retry("parse_failed_then_retry", metadata={"reason": "invalid json"}):
+            sleep_ms(250)
+
+        with trace_model_call(
+            "verify_claims",
+            model="gpt-4.1",
+            input_text=SYSTEM_LONG + "\n".join(summaries),
+            output_tokens=450,
+        ):
+            sleep_ms(450)
+
+        with trace_model_call(
+            "write_report",
+            model="gpt-4.1",
+            input_text=SYSTEM_LONG + "\n".join(summaries),
+            output_tokens=800,
+        ):
+            sleep_ms(550)
+
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path
+
+
+def run_proof_pair(out_dir: str | Path = "runs") -> tuple[Path, Path]:
+    """Generate a before/after trace pair for the closed-loop product demo."""
+
+    return run_proof_baseline(out_dir), run_proof_candidate(out_dir)
+
+
+def run_proof_baseline(out_dir: str | Path = "runs") -> Path:
+    out_path = Path(out_dir) / "agentloop_proof_baseline.json"
+    repeated_context = SYSTEM_LONG + "Classify and summarize every source independently. " * 20
+    with trace_agent(
+        "agentloop_proof_baseline",
+        metadata=_demo_metadata(schema_valid=False, quality_score=0.82),
+    ) as trace:
+        with trace_model_call(
+            "classify_intent",
+            model="gpt-4.1",
+            input_text="Classify this simple support request.",
+            output_tokens=40,
+        ):
+            sleep_ms(220)
+
+        with trace_tool_call("search_web", metadata={"query": "agent workflow profiler"}):
+            sources = fake_search("agent workflow profiler")[:3]
+
+        summaries = []
+        for source in sources:
+            with trace_tool_call("fake_read", metadata={"source": source}):
+                fake_read(source)
+
+        for index in range(15):
+            with trace_model_call(
+                "summarize_source",
+                model="gpt-4.1-mini",
+                input_text=repeated_context + f" item {index}",
+                input_tokens=80,
+                output_tokens=60,
+            ):
+                sleep_ms(12)
+                summaries.append(f"summary {index}")
+
+        for index in range(8):
+            with trace_tool_call("plan_next", metadata={"iteration": index}):
+                sleep_ms(60)
+
+        for name in ["search_web", "inspect_result", "search_web", "inspect_result"]:
+            with trace_tool_call(name):
+                if name == "search_web":
+                    fake_search("same query")
+                else:
+                    inspect_result("same result")
+
+        with trace_retry("invalid_json_retry", metadata={"reason": "schema_validation_failed"}):
+            sleep_ms(180)
+
+        large_context = repeated_context + "\n".join(summaries) + (" Evidence chunk. " * 1600)
+        with trace_model_call(
+            "final_answer",
+            model="gpt-4.1",
+            input_text=large_context,
+            input_tokens=4000,
+            output_tokens=700,
+        ):
+            sleep_ms(500)
+
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path
+
+
+def run_proof_candidate(out_dir: str | Path = "runs") -> Path:
+    out_path = Path(out_dir) / "agentloop_proof_candidate.json"
+    compact_context = (
+        SYSTEM_SHORT + "Use cached instructions, batch summaries, and stop after convergence."
+    )
+    with trace_agent(
+        "agentloop_proof_candidate",
+        metadata=_demo_metadata(schema_valid=True, quality_score=0.94),
+    ) as trace:
+        with trace_model_call(
+            "classify_intent",
+            model="gpt-4.1-mini",
+            input_text="Classify this simple support request.",
+            output_tokens=30,
+        ):
+            sleep_ms(90)
+
+        with trace_tool_call("search_web", metadata={"query": "agent workflow profiler"}):
+            sources = fake_search("agent workflow profiler")[:3]
+
+        with trace_tool_call("fake_read_parallel", metadata={"count": len(sources)}):
+            with ThreadPoolExecutor(max_workers=3) as pool:
+                contents = list(pool.map(fake_read, sources))
+
+        with trace_model_call(
+            "summarize_batch",
+            model="gpt-4.1-mini",
+            input_text=compact_context + "\n".join(content[:220] for content in contents),
+            output_tokens=280,
+        ):
+            sleep_ms(170)
+
+        with trace_tool_call(
+            "plan_next", metadata={"iteration": 1, "stop_reason": "state_converged"}
+        ):
+            sleep_ms(60)
+
+        with trace_model_call(
+            "final_answer",
+            model="gpt-4.1-mini",
+            input_text=compact_context + " Compressed evidence with source ids.",
+            output_tokens=420,
+        ):
+            sleep_ms(220)
+
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path
+
+
+def run_optimized(out_dir: str | Path = "runs") -> Path:
+    out_path = Path(out_dir) / "research_agent_optimized.json"
+    with trace_agent("research_agent_optimized", metadata=_demo_metadata()) as trace:
+        with trace_model_call(
+            "plan",
+            model="gpt-4.1-mini",
+            input_text=SYSTEM_SHORT + "Plan research on agent inference runtimes.",
+            output_tokens=120,
+        ):
+            sleep_ms(200)
+
+        with trace_tool_call("search_web", metadata={"query": "agent inference runtime"}):
+            sources = fake_search("agent inference runtime")
+
+        with trace_tool_call("read_sources_parallel", metadata={"count": len(sources)}):
+            with ThreadPoolExecutor(max_workers=5) as pool:
+                contents = list(pool.map(fake_read, sources))
+
+        compact_context = "\n".join(content[:300] for content in contents)
+        with trace_model_call(
+            "summarize_batch",
+            model="gpt-4.1-mini",
+            input_text=SYSTEM_SHORT + compact_context,
+            output_tokens=500,
+        ):
+            sleep_ms(300)
+
+        with trace_model_call(
+            "verify_and_write",
+            model="gpt-4.1",
+            input_text=SYSTEM_SHORT + compact_context,
+            output_tokens=650,
+        ):
+            sleep_ms(450)
+
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path
+
+
+def run_langgraph_style(out_dir: str | Path = "runs") -> Path:
+    from agentloop.integrations.langgraph import trace_node
+
+    @trace_node("retrieve_sources")
+    def retrieve_sources(state: dict) -> dict:
+        sleep_ms(250)
+        state["sources"] = ["paper-a", "paper-b", "paper-c"]
+        return state
+
+    @trace_node("rank_sources")
+    def rank_sources(state: dict) -> dict:
+        sleep_ms(180)
+        state["sources"] = state["sources"][:2]
+        return state
+
+    out_path = Path(out_dir) / "langgraph_style_demo.json"
+    with trace_agent("langgraph_style_demo", metadata=_demo_metadata()) as trace:
+        state = {"question": "How should agent runtimes be profiled?"}
+        with trace_model_call(
+            "planner", model="gpt-4.1-mini", input_text=state["question"], output_tokens=120
+        ):
+            sleep_ms(160)
+        state = retrieve_sources(state)
+        state = rank_sources(state)
+        with trace_model_call(
+            "final_answer", model="gpt-4.1-mini", input_text=str(state), output_tokens=300
+        ):
+            sleep_ms(240)
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path