agentwatch-dev 0.1.0__py3-none-any.whl

agentwatch/__init__.py

@@ -0,0 +1,74 @@
+"""AgentWatch — predict when production AI agents degrade before they visibly fail.
+
+Quick start:
+
+    import agentwatch
+    agentwatch.configure(api_key="aw_live_...", agent_id="<agent-uuid>")
+
+    # Option A — auto-instrument any OpenAI-compatible client (Groq / Cerebras / OpenAI):
+    from openai import OpenAI
+    client = OpenAI(base_url="https://api.groq.com/openai/v1", api_key=GROQ_KEY)
+    agentwatch.instrument_openai(client)   # every chat completion now emits a trace
+
+    # Option B — wrap your agent function:
+    @agentwatch.watch()
+    def run_agent(query: str) -> str:
+        ...
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, Optional
+
+from .client import AgentWatchClient
+
+__version__ = "0.1.0"
+__all__ = [
+    "AgentWatchClient",
+    "configure",
+    "watch",
+    "instrument_openai",
+    "trace",
+    "__version__",
+]
+
+_default_client: Optional[AgentWatchClient] = None
+
+
+def configure(api_key: str, agent_id: str, **kwargs: Any) -> AgentWatchClient:
+    """Set the process-wide default client used by the module-level helpers."""
+    global _default_client
+    _default_client = AgentWatchClient(api_key=api_key, agent_id=agent_id, **kwargs)
+    return _default_client
+
+
+def _require_default() -> AgentWatchClient:
+    if _default_client is None:
+        raise RuntimeError(
+            "AgentWatch is not configured. Call "
+            "agentwatch.configure(api_key=..., agent_id=...) first, "
+            "or pass client=AgentWatchClient(...) explicitly."
+        )
+    return _default_client
+
+
+def watch(
+    fn: Optional[Callable] = None,
+    *,
+    client: Optional[AgentWatchClient] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Callable:
+    """Wrap a sync/async agent function. Use `@agentwatch.watch()` or `watch(fn)`."""
+    return (client or _require_default()).watch(fn, metadata=metadata)
+
+
+def instrument_openai(
+    openai_client: Any, *, client: Optional[AgentWatchClient] = None
+) -> Any:
+    """Auto-instrument an OpenAI-compatible client (OpenAI / Groq / Cerebras / …)."""
+    return (client or _require_default()).instrument_openai(openai_client)
+
+
+def trace(*, client: Optional[AgentWatchClient] = None, **kwargs: Any) -> None:
+    """Manually submit a trace via the default (or given) client."""
+    (client or _require_default()).trace(**kwargs)

agentwatch/_extract.py

@@ -0,0 +1,105 @@
+"""Helpers to pull trace fields out of LLM responses.
+
+Handles both the official `openai` client's pydantic objects (attribute access)
+and plain dicts (key access), so the same code covers OpenAI, Groq, Cerebras,
+Together, vLLM, and anyone returning an OpenAI-compatible shape.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional
+
+
+def _get(obj: Any, key: str) -> Any:
+    """Read `key` from a dict (by key) or an object (by attribute)."""
+    if obj is None:
+        return None
+    if isinstance(obj, dict):
+        return obj.get(key)
+    return getattr(obj, key, None)
+
+
+def truncate(value: str, max_length: int) -> str:
+    if len(value) <= max_length:
+        return value
+    return value[:max_length] + "…[truncated]"
+
+
+def safe_serialize(value: Any, max_length: int) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return truncate(value, max_length)
+    try:
+        return truncate(json.dumps(value, default=str), max_length)
+    except Exception:
+        return truncate(str(value), max_length)
+
+
+def extract_output(result: Any, max_length: int) -> str:
+    """Prefer the assistant message text; fall back to a serialized dump."""
+    choices = _get(result, "choices")
+    if choices:
+        try:
+            message = _get(choices[0], "message")
+            content = _get(message, "content")
+            if isinstance(content, str):
+                return truncate(content, max_length)
+        except Exception:
+            pass
+    if isinstance(result, str):
+        return truncate(result, max_length)
+    return safe_serialize(result, max_length)
+
+
+def extract_model(result: Any) -> str:
+    model = _get(result, "model")
+    return model if isinstance(model, str) and model else "unknown"
+
+
+def extract_tokens(result: Any) -> Dict[str, int]:
+    usage = _get(result, "usage")
+    if usage is None:
+        return {}
+    out: Dict[str, int] = {}
+    prompt = _get(usage, "prompt_tokens")
+    completion = _get(usage, "completion_tokens")
+    total = _get(usage, "total_tokens")
+    if isinstance(prompt, int):
+        out["promptTokens"] = prompt
+    if isinstance(completion, int):
+        out["completionTokens"] = completion
+    if isinstance(total, int):
+        out["totalTokens"] = total
+    return out
+
+
+def extract_tools(result: Any) -> List[str]:
+    choices = _get(result, "choices")
+    if not choices:
+        return []
+    message = _get(choices[0], "message")
+    tool_calls = _get(message, "tool_calls")
+    if not tool_calls:
+        return []
+    names: List[str] = []
+    for call in tool_calls:
+        fn = _get(call, "function")
+        name = _get(fn, "name")
+        if isinstance(name, str):
+            names.append(name)
+    return names
+
+
+def input_from_messages(messages: Any, max_length: int) -> str:
+    """Represent a chat request by its last user message (most descriptive)."""
+    try:
+        for msg in reversed(list(messages)):
+            if _get(msg, "role") == "user":
+                content = _get(msg, "content")
+                if isinstance(content, str):
+                    return truncate(content, max_length)
+    except Exception:
+        pass
+    return safe_serialize(messages, max_length)

agentwatch/_transport.py

@@ -0,0 +1,38 @@
+"""Fire-and-forget trace delivery.
+
+Monitoring must never surface an error to the production caller and must never
+add latency to its request path, so every trace is POSTed on a daemon thread and
+all exceptions are swallowed.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import urllib.request
+from typing import Any, Dict
+
+SDK_VERSION = "0.1.0"
+
+
+def post_trace(endpoint: str, api_key: str, payload: Dict[str, Any], timeout: float) -> None:
+    def _send() -> None:
+        try:
+            data = json.dumps(payload).encode("utf-8")
+            request = urllib.request.Request(
+                f"{endpoint}/v1/trace",
+                data=data,
+                method="POST",
+                headers={
+                    "Authorization": f"Bearer {api_key}",
+                    "Content-Type": "application/json",
+                    "X-SDK-Version": SDK_VERSION,
+                    "User-Agent": f"agentwatch-python/{SDK_VERSION}",
+                },
+            )
+            urllib.request.urlopen(request, timeout=timeout).close()
+        except Exception:
+            # Intentional no-op: telemetry failures must not affect the caller.
+            pass
+
+    threading.Thread(target=_send, daemon=True).start()

agentwatch/client.py

@@ -0,0 +1,283 @@
+"""AgentWatchClient — the stateful entry point.
+
+Mirrors the TypeScript SDK's clean-client design: it only captures a trace and
+POSTs it to the ingestion API. All drift math (CUSUM/EWMA/judge/scoring) runs
+server-side, so this package ships none of the engine.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import time
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, List, Optional
+
+from . import _extract
+from ._transport import post_trace
+
+DEFAULT_ENDPOINT = "https://api.agentwatch.dev"
+DEFAULT_MAX_PREVIEW = 5000
+DEFAULT_TIMEOUT = 5.0  # seconds
+MAX_LATENCY_MS = 600_000
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _elapsed_ms(start: float) -> float:
+    return (time.perf_counter() - start) * 1000.0
+
+
+class AgentWatchClient:
+    """Reusable client for a single agent.
+
+    Example:
+        client = AgentWatchClient(api_key="aw_live_...", agent_id="<uuid>")
+        run = client.watch(run_agent)          # wrap a function
+        client.instrument_openai(openai_client)  # or auto-instrument a client
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        agent_id: str,
+        endpoint: Optional[str] = None,
+        max_preview_length: int = DEFAULT_MAX_PREVIEW,
+        timeout: float = DEFAULT_TIMEOUT,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        if not api_key:
+            raise ValueError("AgentWatch: api_key is required")
+        if not agent_id:
+            raise ValueError("AgentWatch: agent_id is required")
+        self.api_key = api_key
+        self.agent_id = agent_id
+        self.endpoint = (endpoint or DEFAULT_ENDPOINT).rstrip("/")
+        self.max_preview_length = max_preview_length
+        self.timeout = timeout
+        self.metadata: Dict[str, Any] = dict(metadata or {})
+
+    # ── internals ────────────────────────────────────────────────────────────
+
+    def _dispatch(
+        self,
+        *,
+        input_str: str,
+        output_str: str,
+        model: str,
+        latency_ms: float,
+        status: str,
+        tools_used: Optional[List[str]] = None,
+        tokens: Optional[Dict[str, int]] = None,
+        error: Optional[str] = None,
+        session_id: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        payload: Dict[str, Any] = {
+            "agentId": self.agent_id,
+            "sessionId": session_id or str(uuid.uuid4()),
+            "input": _extract.truncate(input_str, self.max_preview_length),
+            "output": _extract.truncate(output_str, self.max_preview_length),
+            "model": model or "unknown",
+            "latencyMs": max(0, min(MAX_LATENCY_MS, int(round(latency_ms)))),
+            "status": status,
+            "timestamp": _now_iso(),
+        }
+        merged_meta = {**self.metadata, **(metadata or {})}
+        if merged_meta:
+            payload["metadata"] = merged_meta
+        if tools_used:
+            payload["toolsUsed"] = tools_used
+        if tokens:
+            payload.update({k: v for k, v in tokens.items() if v is not None})
+        if error:
+            payload["error"] = _extract.truncate(error, 2000)
+        post_trace(self.endpoint, self.api_key, payload, self.timeout)
+
+    @staticmethod
+    def _input_of(args: tuple, kwargs: dict) -> Any:
+        if args:
+            return args[0]
+        if kwargs:
+            return next(iter(kwargs.values()))
+        return None
+
+    def _dispatch_success(
+        self, result: Any, args: tuple, kwargs: dict, start: float, metadata: Optional[Dict[str, Any]]
+    ) -> None:
+        tools = _extract.extract_tools(result)
+        self._dispatch(
+            input_str=_extract.safe_serialize(self._input_of(args, kwargs), self.max_preview_length),
+            output_str=_extract.extract_output(result, self.max_preview_length),
+            model=_extract.extract_model(result),
+            latency_ms=_elapsed_ms(start),
+            status="success",
+            tools_used=tools or None,
+            tokens=_extract.extract_tokens(result),
+            metadata=metadata,
+        )
+
+    # ── public API ───────────────────────────────────────────────────────────
+
+    def trace(
+        self,
+        *,
+        input: Any,
+        output: Any,
+        latency_ms: float,
+        model: str = "unknown",
+        status: str = "success",
+        tools_used: Optional[List[str]] = None,
+        prompt_tokens: Optional[int] = None,
+        completion_tokens: Optional[int] = None,
+        total_tokens: Optional[int] = None,
+        error: Optional[str] = None,
+        session_id: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Manually submit a trace — for custom integrations you can't wrap."""
+        tokens = {
+            "promptTokens": prompt_tokens,
+            "completionTokens": completion_tokens,
+            "totalTokens": total_tokens,
+        }
+        self._dispatch(
+            input_str=_extract.safe_serialize(input, self.max_preview_length),
+            output_str=_extract.safe_serialize(output, self.max_preview_length),
+            model=model,
+            latency_ms=latency_ms,
+            status=status,
+            tools_used=tools_used,
+            tokens={k: v for k, v in tokens.items() if v is not None},
+            error=error,
+            session_id=session_id,
+            metadata=metadata,
+        )
+
+    def watch(
+        self,
+        fn: Optional[Callable] = None,
+        *,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> Callable:
+        """Wrap a (sync or async) agent function to auto-capture traces.
+
+        Usable as `client.watch(fn)` or as a decorator `@client.watch()`.
+        Telemetry is fire-and-forget; errors are captured then re-raised so the
+        caller's behavior is unchanged.
+        """
+
+        def decorator(func: Callable) -> Callable:
+            if asyncio.iscoroutinefunction(func):
+
+                @functools.wraps(func)
+                async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
+                    start = time.perf_counter()
+                    try:
+                        result = await func(*args, **kwargs)
+                    except Exception as err:
+                        self._dispatch(
+                            input_str=_extract.safe_serialize(
+                                self._input_of(args, kwargs), self.max_preview_length
+                            ),
+                            output_str="",
+                            model="unknown",
+                            latency_ms=_elapsed_ms(start),
+                            status="error",
+                            error=str(err),
+                            metadata=metadata,
+                        )
+                        raise
+                    self._dispatch_success(result, args, kwargs, start, metadata)
+                    return result
+
+                return async_wrapper
+
+            @functools.wraps(func)
+            def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
+                start = time.perf_counter()
+                try:
+                    result = func(*args, **kwargs)
+                except Exception as err:
+                    self._dispatch(
+                        input_str=_extract.safe_serialize(
+                            self._input_of(args, kwargs), self.max_preview_length
+                        ),
+                        output_str="",
+                        model="unknown",
+                        latency_ms=_elapsed_ms(start),
+                        status="error",
+                        error=str(err),
+                        metadata=metadata,
+                    )
+                    raise
+                self._dispatch_success(result, args, kwargs, start, metadata)
+                return result
+
+            return sync_wrapper
+
+        return decorator(fn) if fn is not None else decorator
+
+    def instrument_openai(self, openai_client: Any) -> Any:
+        """Monkeypatch an OpenAI-compatible client so every chat completion emits
+        a trace automatically — one line, no per-call wrapping.
+
+        Works for OpenAI, Groq, Cerebras, Together, vLLM — anything you drive
+        through the `openai` client with a different base_url. Idempotent.
+        Returns the same client for chaining.
+        """
+        completions = openai_client.chat.completions
+        if getattr(completions, "_agentwatch_instrumented", False):
+            return openai_client
+
+        original_create = completions.create
+        watcher = self
+
+        @functools.wraps(original_create)
+        def wrapped_create(*args: Any, **kwargs: Any) -> Any:
+            start = time.perf_counter()
+            messages = kwargs.get("messages")
+            input_str = (
+                _extract.input_from_messages(messages, watcher.max_preview_length)
+                if messages is not None
+                else ""
+            )
+            try:
+                response = original_create(*args, **kwargs)
+            except Exception as err:
+                watcher._dispatch(
+                    input_str=input_str,
+                    output_str="",
+                    model=str(kwargs.get("model", "unknown")),
+                    latency_ms=_elapsed_ms(start),
+                    status="error",
+                    error=str(err),
+                )
+                raise
+            watcher._dispatch(
+                input_str=input_str,
+                output_str=_extract.extract_output(response, watcher.max_preview_length),
+                model=_extract.extract_model(response) or str(kwargs.get("model", "unknown")),
+                latency_ms=_elapsed_ms(start),
+                status="success",
+                tools_used=_extract.extract_tools(response) or None,
+                tokens=_extract.extract_tokens(response),
+            )
+            return response
+
+        wrapped_create._agentwatch_original = original_create  # type: ignore[attr-defined]
+        completions.create = wrapped_create  # type: ignore[method-assign]
+        completions._agentwatch_instrumented = True  # type: ignore[attr-defined]
+        return openai_client
+
+    def uninstrument_openai(self, openai_client: Any) -> Any:
+        """Undo instrument_openai() (mainly for tests)."""
+        completions = openai_client.chat.completions
+        original = getattr(completions.create, "_agentwatch_original", None)
+        if original is not None:
+            completions.create = original  # type: ignore[method-assign]
+            completions._agentwatch_instrumented = False  # type: ignore[attr-defined]
+        return openai_client

agentwatch_dev-0.1.0.dist-info/METADATA

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: agentwatch-dev
+Version: 0.1.0
+Summary: Predict when production AI agents degrade — before they visibly fail. Python SDK.
+Project-URL: Homepage, https://agentwatch.dev
+Author: AgentWatch
+License: MIT
+License-File: LICENSE
+Keywords: ai-agents,cerebras,drift,groq,llm,monitoring,observability,openai
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# AgentWatch — Python SDK
+
+**Predict when production AI agents are degrading — before they visibly fail.**
+
+AgentWatch watches your agent's traces and runs statistical change-point
+detection (CUSUM/EWMA), an LLM-judge correctness gate, and semantic-drift scoring
+**server-side** to catch quality collapse *before* your error rate moves. This SDK
+just captures traces and ships them — it has **zero required dependencies** and
+never adds latency to or raises errors in your agent's request path.
+
+## Install
+
+```bash
+pip install agentwatch-dev
+# optional, only for instrument_openai():
+pip install "agentwatch-dev[openai]"
+```
+
+The distribution is named `agentwatch-dev` (after agentwatch.dev); the import name
+is just `agentwatch`.
+
+## Configure
+
+```python
+import agentwatch
+agentwatch.configure(api_key="aw_live_...", agent_id="<agent-uuid>")
+```
+
+Self-hosting? Pass `endpoint="https://your-host"` to `configure(...)`.
+
+## Option A — auto-instrument (one line)
+
+Works for **any OpenAI-compatible client** — OpenAI, Groq, Cerebras, Together,
+vLLM — because they all use the `openai` client with a different `base_url`.
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="https://api.groq.com/openai/v1", api_key=GROQ_API_KEY)
+agentwatch.instrument_openai(client)
+
+# Every chat completion from here on is traced automatically:
+client.chat.completions.create(
+    model="llama-3.3-70b-versatile",
+    messages=[{"role": "user", "content": "Refund order #47829"}],
+)
+```
+
+## Option B — wrap your agent function
+
+```python
+@agentwatch.watch()
+def run_agent(query: str) -> str:
+    ...
+    return answer
+
+# async is supported too:
+@agentwatch.watch()
+async def run_agent_async(query: str): ...
+```
+
+`watch()` captures the first argument as the input and the return value as the
+output. If the return value is an OpenAI-shaped response, it also extracts the
+model, token counts, and tool names automatically.
+
+## Option C — manual trace
+
+```python
+client = agentwatch.AgentWatchClient(api_key="aw_live_...", agent_id="<uuid>")
+client.trace(
+    input="Refund order #47829",
+    output="I've issued the refund…",
+    model="llama-3.3-70b-versatile",
+    latency_ms=812,
+    prompt_tokens=220, completion_tokens=180, total_tokens=400,
+    tools_used=["search_kb", "issue_refund"],
+)
+```
+
+## Notes
+
+- **Fire-and-forget:** traces are POSTed on a daemon thread; delivery failures are
+  swallowed so monitoring can never break your agent.
+- **Metadata:** attach tags via `configure(..., metadata={"env": "prod"})` or
+  per-call `@agentwatch.watch(metadata={"version": "2.1"})`.
+- **Streaming** (`stream=True`) responses aren't fully captured yet — the trace is
+  still recorded, but token/output extraction may be partial.
+
+MIT licensed.

agentwatch_dev-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+agentwatch/__init__.py,sha256=eaSLdz6HQGADR1r1NqVMr4yWroHvN2SO1xFO7z7kaDs,2423
+agentwatch/_extract.py,sha256=f7y-CEqsdUsqwOFwzOm-pxEagM9n1oRwnNOxTMyRFrw,3350
+agentwatch/_transport.py,sha256=37LWah05wSAoNHmGEQiDDcx5TH4eUda-dJdhcMa4lkA,1283
+agentwatch/client.py,sha256=rVPMVoaLLVT_KFM2G9rcqjeB3G5yP3fNgQ0VjWaQTDk,11234
+agentwatch_dev-0.1.0.dist-info/METADATA,sha256=p4qFOugmaNUgUoZg4oHRpu9EwXAsH0O_gFJIfPwWGtg,3429
+agentwatch_dev-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+agentwatch_dev-0.1.0.dist-info/licenses/LICENSE,sha256=sNFI0vIuwEXtlgwJktNQrqEh55kX9p63RuwnHX8Z0ps,1066
+agentwatch_dev-0.1.0.dist-info/RECORD,,

agentwatch_dev-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

agentwatch_dev-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 AgentWatch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALING IN THE
+SOFTWARE.