agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/demo.py

@@ -0,0 +1,300 @@
+from __future__ import annotations
+
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from agentloop import trace_agent, trace_model_call, trace_retry, trace_tool_call
+
+# Every model call below declares both token counts explicitly. These traces are
+# synthetic fixtures, so their counts are declared fixture values the way a real
+# integration declares provider-reported usage — not measurements of the prompt
+# text. Leaving a count off would make AgentLoop fall back to a whitespace word
+# estimate, which records `estimated_words` provenance and correctly turns the
+# demo's cost gates indeterminate (issue #133).
+SYSTEM_LONG = (
+    "You are a careful research agent. Use evidence, verify claims, and write concise reports. "
+    * 20
+)
+SYSTEM_SHORT = (
+    "You are a careful research agent. Use evidence, verify claims, and write concise reports. " * 4
+)
+
+
+def _demo_metadata(**values: object) -> dict[str, object]:
+    metadata: dict[str, object] = {"synthetic": True, "source": "agentloop_demo"}
+    metadata.update(values)
+    return metadata
+
+
+def sleep_ms(ms: int) -> None:
+    time.sleep(ms / 1000)
+
+
+def fake_search(query: str) -> list[str]:
+    sleep_ms(300)
+    return [f"source-{i}-{query}" for i in range(1, 6)]
+
+
+def fake_read(source: str) -> str:
+    sleep_ms(250)
+    return f"Content from {source}. " * 30
+
+
+def inspect_result(result: str) -> str:
+    sleep_ms(120)
+    return f"inspected {result}"
+
+
+def run_baseline(out_dir: str | Path = "runs") -> Path:
+    out_path = Path(out_dir) / "research_agent_baseline.json"
+    with trace_agent("research_agent_baseline", metadata=_demo_metadata()) as trace:
+        with trace_model_call(
+            "plan",
+            model="gpt-4.1",
+            input_text=SYSTEM_LONG + "Plan research on agent inference runtimes.",
+            input_tokens=286,
+            output_tokens=220,
+        ):
+            sleep_ms(400)
+
+        with trace_tool_call("search_web", metadata={"query": "agent inference runtime"}):
+            sources = fake_search("agent inference runtime")
+
+        summaries = []
+        for source in sources:
+            with trace_tool_call("read_source", metadata={"source": source}):
+                content = fake_read(source)
+            with trace_model_call(
+                "summarize_source",
+                model="gpt-4.1",
+                input_text=SYSTEM_LONG + content,
+                input_tokens=430,
+                output_tokens=350,
+            ):
+                sleep_ms(350)
+                summaries.append("summary for " + source)
+
+        with trace_retry("parse_failed_then_retry", metadata={"reason": "invalid json"}):
+            sleep_ms(250)
+
+        with trace_model_call(
+            "verify_claims",
+            model="gpt-4.1",
+            input_text=SYSTEM_LONG + "\n".join(summaries),
+            input_tokens=305,
+            output_tokens=450,
+        ):
+            sleep_ms(450)
+
+        with trace_model_call(
+            "write_report",
+            model="gpt-4.1",
+            input_text=SYSTEM_LONG + "\n".join(summaries),
+            input_tokens=305,
+            output_tokens=800,
+        ):
+            sleep_ms(550)
+
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path
+
+
+def run_proof_pair(out_dir: str | Path = "runs") -> tuple[Path, Path]:
+    """Generate a before/after trace pair for the closed-loop product demo."""
+
+    return run_proof_baseline(out_dir), run_proof_candidate(out_dir)
+
+
+def run_proof_baseline(out_dir: str | Path = "runs") -> Path:
+    out_path = Path(out_dir) / "agentloop_proof_baseline.json"
+    repeated_context = SYSTEM_LONG + "Classify and summarize every source independently. " * 20
+    with trace_agent(
+        "agentloop_proof_baseline",
+        metadata=_demo_metadata(schema_valid=False, quality_score=0.82),
+    ) as trace:
+        with trace_model_call(
+            "classify_intent",
+            model="gpt-4.1",
+            input_text="Classify this simple support request.",
+            input_tokens=5,
+            output_tokens=40,
+        ):
+            sleep_ms(220)
+
+        with trace_tool_call("search_web", metadata={"query": "agent workflow profiler"}):
+            sources = fake_search("agent workflow profiler")[:3]
+
+        summaries = []
+        for source in sources:
+            with trace_tool_call("fake_read", metadata={"source": source}):
+                fake_read(source)
+
+        for index in range(15):
+            with trace_model_call(
+                "summarize_source",
+                model="gpt-4.1-mini",
+                input_text=repeated_context + f" item {index}",
+                input_tokens=80,
+                output_tokens=60,
+            ):
+                sleep_ms(12)
+                summaries.append(f"summary {index}")
+
+        for index in range(8):
+            with trace_tool_call("plan_next", metadata={"iteration": index}):
+                sleep_ms(60)
+
+        for name in ["search_web", "inspect_result", "search_web", "inspect_result"]:
+            with trace_tool_call(name):
+                if name == "search_web":
+                    fake_search("same query")
+                else:
+                    inspect_result("same result")
+
+        with trace_retry("invalid_json_retry", metadata={"reason": "schema_validation_failed"}):
+            sleep_ms(180)
+
+        large_context = repeated_context + "\n".join(summaries) + (" Evidence chunk. " * 1600)
+        with trace_model_call(
+            "final_answer",
+            model="gpt-4.1",
+            input_text=large_context,
+            input_tokens=4000,
+            output_tokens=700,
+        ):
+            sleep_ms(500)
+
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path
+
+
+def run_proof_candidate(out_dir: str | Path = "runs") -> Path:
+    out_path = Path(out_dir) / "agentloop_proof_candidate.json"
+    compact_context = (
+        SYSTEM_SHORT + "Use cached instructions, batch summaries, and stop after convergence."
+    )
+    with trace_agent(
+        "agentloop_proof_candidate",
+        metadata=_demo_metadata(schema_valid=True, quality_score=0.94),
+    ) as trace:
+        with trace_model_call(
+            "classify_intent",
+            model="gpt-4.1-mini",
+            input_text="Classify this simple support request.",
+            input_tokens=5,
+            output_tokens=30,
+        ):
+            sleep_ms(90)
+
+        with trace_tool_call("search_web", metadata={"query": "agent workflow profiler"}):
+            sources = fake_search("agent workflow profiler")[:3]
+
+        with trace_tool_call("fake_read_parallel", metadata={"count": len(sources)}):
+            with ThreadPoolExecutor(max_workers=3) as pool:
+                contents = list(pool.map(fake_read, sources))
+
+        with trace_model_call(
+            "summarize_batch",
+            model="gpt-4.1-mini",
+            input_text=compact_context + "\n".join(content[:220] for content in contents),
+            input_tokens=136,
+            output_tokens=280,
+        ):
+            sleep_ms(170)
+
+        with trace_tool_call(
+            "plan_next", metadata={"iteration": 1, "stop_reason": "state_converged"}
+        ):
+            sleep_ms(60)
+
+        with trace_model_call(
+            "final_answer",
+            model="gpt-4.1-mini",
+            input_text=compact_context + " Compressed evidence with source ids.",
+            input_tokens=70,
+            output_tokens=420,
+        ):
+            sleep_ms(220)
+
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path
+
+
+def run_optimized(out_dir: str | Path = "runs") -> Path:
+    out_path = Path(out_dir) / "research_agent_optimized.json"
+    with trace_agent("research_agent_optimized", metadata=_demo_metadata()) as trace:
+        with trace_model_call(
+            "plan",
+            model="gpt-4.1-mini",
+            input_text=SYSTEM_SHORT + "Plan research on agent inference runtimes.",
+            input_tokens=62,
+            output_tokens=120,
+        ):
+            sleep_ms(200)
+
+        with trace_tool_call("search_web", metadata={"query": "agent inference runtime"}):
+            sources = fake_search("agent inference runtime")
+
+        with trace_tool_call("read_sources_parallel", metadata={"count": len(sources)}):
+            with ThreadPoolExecutor(max_workers=5) as pool:
+                contents = list(pool.map(fake_read, sources))
+
+        compact_context = "\n".join(content[:300] for content in contents)
+        with trace_model_call(
+            "summarize_batch",
+            model="gpt-4.1-mini",
+            input_text=SYSTEM_SHORT + compact_context,
+            input_tokens=221,
+            output_tokens=500,
+        ):
+            sleep_ms(300)
+
+        with trace_model_call(
+            "verify_and_write",
+            model="gpt-4.1",
+            input_text=SYSTEM_SHORT + compact_context,
+            input_tokens=221,
+            output_tokens=650,
+        ):
+            sleep_ms(450)
+
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path
+
+
+def run_langgraph_style(out_dir: str | Path = "runs") -> Path:
+    from agentloop.integrations.langgraph import trace_node
+
+    @trace_node("retrieve_sources")
+    def retrieve_sources(state: dict) -> dict:
+        sleep_ms(250)
+        state["sources"] = ["paper-a", "paper-b", "paper-c"]
+        return state
+
+    @trace_node("rank_sources")
+    def rank_sources(state: dict) -> dict:
+        sleep_ms(180)
+        state["sources"] = state["sources"][:2]
+        return state
+
+    out_path = Path(out_dir) / "langgraph_style_demo.json"
+    with trace_agent("langgraph_style_demo", metadata=_demo_metadata()) as trace:
+        state = {"question": "How should agent runtimes be profiled?"}
+        with trace_model_call(
+            "planner", model="gpt-4.1-mini", input_text=state["question"], output_tokens=120
+        ):
+            sleep_ms(160)
+        state = retrieve_sources(state)
+        state = rank_sources(state)
+        with trace_model_call(
+            "final_answer", model="gpt-4.1-mini", input_text=str(state), output_tokens=300
+        ):
+            sleep_ms(240)
+    trace.export_json(out_path)
+    trace.print_report()
+    return out_path

agentloop/doctor.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import importlib
+import os
+import platform
+import socket
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.error import URLError
+from urllib.parse import urlsplit
+from urllib.request import Request, urlopen
+
+from agentloop.config import postgres_connection_source
+from agentloop.runtime import get_runtime_config
+from agentloop.store import get_store
+
+
+@dataclass
+class DoctorCheck:
+    name: str
+    status: str
+    detail: str
+    fix: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = {"name": self.name, "status": self.status, "detail": self.detail}
+        if self.fix:
+            payload["fix"] = self.fix
+        return payload
+
+
+def _ok(name: str, detail: str) -> DoctorCheck:
+    return DoctorCheck(name=name, status="ok", detail=detail)
+
+
+def _warn(name: str, detail: str, fix: str | None = None) -> DoctorCheck:
+    return DoctorCheck(name=name, status="warn", detail=detail, fix=fix)
+
+
+def _fail(name: str, detail: str, fix: str | None = None) -> DoctorCheck:
+    return DoctorCheck(name=name, status="fail", detail=detail, fix=fix)
+
+
+def _can_connect(host: str, port: int, timeout: float = 1.0) -> bool:
+    try:
+        with socket.create_connection((host, port), timeout=timeout):
+            return True
+    except OSError:
+        return False
+
+
+def _health_check(api_url: str, timeout: float = 2.0) -> DoctorCheck:
+    parsed = urlsplit(api_url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        return _warn(
+            "api_health",
+            "AGENTLOOP_API_URL must use http:// or https://",
+            "Set AGENTLOOP_API_URL to the AgentLoop HTTP API origin.",
+        )
+    url = api_url.rstrip("/") + "/health"
+    try:
+        request = Request(url, headers={"User-Agent": "agentloop-doctor"})
+        with urlopen(  # nosec B310 - API URL scheme is validated above.
+            request, timeout=timeout
+        ) as response:
+            if response.status == 200:
+                return _ok("api_health", f"{url} returned HTTP 200")
+            return _warn("api_health", f"{url} returned HTTP {response.status}")
+    except URLError as exc:
+        return _warn(
+            "api_health",
+            f"Could not reach {url}: {exc.reason}",
+            "Start the API with: agentloop server --host 127.0.0.1 --port 8000",
+        )
+    except Exception as exc:  # pragma: no cover - defensive for platform SSL/socket edge cases
+        return _warn("api_health", f"Could not reach {url}: {exc}")
+
+
+def _ready_check(api_url: str, timeout: float = 2.0) -> DoctorCheck:
+    parsed = urlsplit(api_url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        return _fail(
+            "api_ready",
+            "AGENTLOOP_API_URL must use http:// or https://",
+            "Set AGENTLOOP_API_URL to the deployed AgentLoop HTTP API origin.",
+        )
+    url = api_url.rstrip("/") + "/readyz"
+    try:
+        request = Request(url, headers={"User-Agent": "agentloop-production-check"})
+        with urlopen(  # nosec B310 - API URL scheme is validated above.
+            request, timeout=timeout
+        ) as response:
+            if response.status == 200:
+                return _ok("api_ready", f"{url} returned HTTP 200")
+            return _fail("api_ready", f"{url} returned HTTP {response.status}")
+    except URLError as exc:
+        return _fail(
+            "api_ready",
+            f"Could not reach {url}: {exc.reason}",
+            "Deploy the API and confirm /readyz is reachable.",
+        )
+    except Exception as exc:  # pragma: no cover - defensive for platform SSL/socket edge cases
+        return _fail("api_ready", f"Could not reach {url}: {exc}")
+
+
+def _env_enabled(name: str) -> bool:
+    return os.getenv(name, "").lower() in {"1", "true", "yes", "on"}
+
+
+def run_doctor(*, check_api: bool = True, runs_dir: Path | str = "runs") -> dict[str, Any]:
+    """Run local setup checks, probing hosted wiring only when it is configured."""
+
+    checks: list[DoctorCheck] = []
+
+    py_version = sys.version_info
+    if py_version >= (3, 10):
+        checks.append(_ok("python", f"Python {platform.python_version()}"))
+    else:
+        checks.append(
+            _fail(
+                "python",
+                f"Python {platform.python_version()} is too old",
+                "Use Python 3.10 or newer.",
+            )
+        )
+
+    # These are the actual core runtime dependencies in pyproject.toml. Server,
+    # dashboard, Postgres, and instrumentation dependencies are checked by their
+    # corresponding modes instead of making a minimal local install look broken.
+    for module_name in ["typer", "rich"]:
+        try:
+            importlib.import_module(module_name)
+            checks.append(_ok(f"dependency:{module_name}", "importable"))
+        except ImportError:
+            checks.append(
+                _fail(
+                    f"dependency:{module_name}",
+                    "not installed",
+                    "Install the package with: python -m pip install agentloop-profiler",
+                )
+            )
+
+    out_dir = Path(runs_dir)
+    try:
+        out_dir.mkdir(parents=True, exist_ok=True)
+        probe = out_dir / ".agentloop_write_probe"
+        probe.write_text("ok", encoding="utf-8")
+        probe.unlink(missing_ok=True)
+        checks.append(_ok("runs_dir", f"{out_dir} is writable"))
+    except Exception as exc:
+        checks.append(
+            _fail(
+                "runs_dir",
+                f"{out_dir} is not writable: {exc}",
+                "Choose a writable path or set AGENTLOOP_EXPORT_DIR.",
+            )
+        )
+
+    try:
+        db = get_store()
+        db.init()
+        checks.append(_ok("store", f"{db.__class__.__name__} initialized"))
+    except Exception as exc:
+        checks.append(
+            _fail(
+                "store",
+                f"store initialization failed: {exc}",
+                "Check AGENTLOOP_STORE_BACKEND and database credentials.",
+            )
+        )
+
+    cfg = get_runtime_config()
+    auth_required = _env_enabled("AGENTLOOP_REQUIRE_API_KEY")
+    explicit_api_url = os.getenv("AGENTLOOP_API_URL")
+
+    if cfg.auto_upload and auth_required and not cfg.api_key:
+        checks.append(
+            _warn(
+                "api_key",
+                "auto upload is enabled but AGENTLOOP_API_KEY is empty",
+                "Create a key with: agentloop create-api-key --project-id demo",
+            )
+        )
+    elif cfg.api_key:
+        checks.append(_ok("api_key", "AGENTLOOP_API_KEY is configured"))
+    else:
+        checks.append(_ok("api_key", "not configured; not required for local-only use"))
+
+    if cfg.auto_upload:
+        checks.append(_ok("auto_upload", f"enabled -> {cfg.api_url}"))
+    else:
+        checks.append(_ok("auto_upload", "disabled; traces stay local"))
+
+    if check_api:
+        if explicit_api_url or cfg.auto_upload:
+            checks.append(_health_check(cfg.api_url))
+        else:
+            checks.append(_ok("api_health", "skipped; no hosted API is configured"))
+
+    failed = sum(1 for check in checks if check.status == "fail")
+    warned = sum(1 for check in checks if check.status == "warn")
+    return {
+        "ok": failed == 0,
+        "failed": failed,
+        "warnings": warned,
+        "checks": [check.to_dict() for check in checks],
+    }
+
+
+def run_production_check(
+    *,
+    check_api: bool = True,
+    check_store: bool = True,
+    allow_http: bool = False,
+) -> dict[str, Any]:
+    """Run strict checks for an internet-facing AgentLoop API/dashboard deployment."""
+
+    checks: list[DoctorCheck] = []
+
+    py_version = sys.version_info
+    if py_version >= (3, 10):
+        checks.append(_ok("python", f"Python {platform.python_version()}"))
+    else:
+        checks.append(
+            _fail(
+                "python",
+                f"Python {platform.python_version()} is too old",
+                "Use Python 3.10 or newer.",
+            )
+        )
+
+    backend = os.getenv("AGENTLOOP_STORE_BACKEND", "sqlite").lower()
+    if backend == "postgres":
+        checks.append(_ok("store_backend", "postgres"))
+    else:
+        checks.append(
+            _fail(
+                "store_backend",
+                f"{backend} is not production-ready",
+                "Set AGENTLOOP_STORE_BACKEND=postgres.",
+            )
+        )
+
+    database_source = postgres_connection_source()
+    if database_source:
+        checks.append(_ok("database_url", f"configured via {database_source}"))
+    else:
+        checks.append(
+            _fail(
+                "database_url",
+                "missing",
+                "Set AGENTLOOP_DATABASE_URL/DATABASE_URL or libpq PGHOST/PGSERVICE variables.",
+            )
+        )
+
+    if check_store:
+        try:
+            db = get_store()
+            db.init()
+            checks.append(_ok("store", f"{db.__class__.__name__} initialized"))
+        except Exception as exc:
+            checks.append(
+                _fail(
+                    "store",
+                    f"store initialization failed: {exc}",
+                    "Check database credentials and network access.",
+                )
+            )
+
+    if _env_enabled("AGENTLOOP_REQUIRE_API_KEY"):
+        checks.append(_ok("api_auth", "AGENTLOOP_REQUIRE_API_KEY=true"))
+    else:
+        checks.append(
+            _fail("api_auth", "API key auth is disabled", "Set AGENTLOOP_REQUIRE_API_KEY=true.")
+        )
+
+    admin_key = os.getenv("AGENTLOOP_ADMIN_API_KEY", "")
+    weak_admin_values = {
+        "",
+        "change-me-admin-key",
+        "replace-with-a-long-random-admin-secret",
+        "dev-secret",
+        "admin-secret",
+    }
+    if admin_key in weak_admin_values or len(admin_key) < 32:
+        checks.append(
+            _fail(
+                "admin_api_key",
+                "missing or too weak",
+                "Set AGENTLOOP_ADMIN_API_KEY to a generated secret with at least 32 characters.",
+            )
+        )
+    else:
+        checks.append(_ok("admin_api_key", "configured"))
+
+    cors_origins = [
+        origin.strip()
+        for origin in os.getenv("AGENTLOOP_CORS_ORIGINS", "").split(",")
+        if origin.strip()
+    ]
+    if not cors_origins:
+        checks.append(
+            _fail("cors_origins", "missing", "Set AGENTLOOP_CORS_ORIGINS to the dashboard origin.")
+        )
+    elif "*" in cors_origins:
+        checks.append(
+            _fail(
+                "cors_origins",
+                "wildcard origin is not allowed for production",
+                "Use exact HTTPS origins.",
+            )
+        )
+    else:
+        checks.append(_ok("cors_origins", ", ".join(cors_origins)))
+
+    api_url = os.getenv("AGENTLOOP_API_URL", "")
+    if not api_url:
+        checks.append(_fail("api_url", "missing", "Set AGENTLOOP_API_URL to the public API URL."))
+    elif api_url.startswith("https://"):
+        checks.append(_ok("api_url", api_url))
+    elif allow_http:
+        checks.append(
+            _warn("api_url", f"{api_url} is not HTTPS", "Use HTTPS for production traffic.")
+        )
+    else:
+        checks.append(_fail("api_url", f"{api_url} is not HTTPS", "Use an HTTPS public API URL."))
+
+    if check_api and api_url:
+        checks.append(_health_check(api_url))
+        checks.append(_ready_check(api_url))
+
+    failed = sum(1 for check in checks if check.status == "fail")
+    warned = sum(1 for check in checks if check.status == "warn")
+    return {
+        "ok": failed == 0,
+        "failed": failed,
+        "warnings": warned,
+        "checks": [check.to_dict() for check in checks],
+    }