agentx-python 0.8.26__py3-none-any.whl → 0.8.27__py3-none-any.whl

agentx/integrations/_traced_call.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 import asyncio
 import inspect
 import json
+import time
 from typing import Any, Callable, Dict, Optional
 
 from agentx.tracing.tracer import Tracer, _safe_serialize
@@ -107,6 +108,7 @@ def finish_llm_call(
     cache_read_tokens: Optional[int] = None,
     cache_write_tokens: Optional[int] = None,
     tool_definitions: Optional[list] = None,
+    call_metadata: Optional[Dict[str, Any]] = None,
 ) -> None:
     """
     Close out one raw-client LLM call - shared by the ``on_finish``/exit
@@ -150,13 +152,19 @@ def finish_llm_call(
             output_tokens=output_tokens,
             cache_read_tokens=cache_read_tokens,
             cache_write_tokens=cache_write_tokens,
+            metadata=call_metadata,
         )
         return
 
     # A patched provider call outside any active span becomes its own root trace - it is a bare
     # model call, so stamp it "llm" rather than leaving the kind unset.
     span = tracer.trace(
-        name, metadata=metadata, framework=framework, model=model, session_id=session_id, span_kind="llm"
+        name,
+        metadata={**(metadata or {}), **(call_metadata or {})} if (metadata or call_metadata) else None,
+        framework=framework,
+        model=model,
+        session_id=session_id,
+        span_kind="llm",
     )
     span.__enter__()
     span._start = start_t
@@ -173,3 +181,215 @@ def finish_llm_call(
     if cache_write_tokens:
         span._cache_write_tokens = cache_write_tokens
     span.__exit__(None, None, None)
+
+
+# ---------------------------------------------------------------------------
+# Streaming: wrap a provider's chunk stream so the trace is built from what
+# was actually streamed, without touching the caller's consumption of it.
+# ---------------------------------------------------------------------------
+
+class StreamAccumulator:
+    """
+    What a streaming patch feeds each chunk into. Subclasses collect the
+    provider-specific pieces (text deltas, tool-call deltas, the usage block
+    that only arrives on the final chunk) and hand back the finished picture
+    in ``result()``.
+    """
+
+    def feed(self, chunk: Any) -> None:  # pragma: no cover - interface
+        raise NotImplementedError
+
+    def result(self) -> Dict[str, Any]:  # pragma: no cover - interface
+        raise NotImplementedError
+
+
+class TracedStream:
+    """
+    Transparent proxy over a provider ``Stream``/``AsyncStream``: iterates the
+    real object, feeds every chunk to the accumulator, and calls ``on_finish``
+    exactly once when the stream is exhausted, raises, is closed (``close()``,
+    ``with``/``async with`` exit), or is dropped part-way and garbage
+    collected - so an abandoned stream still records what it streamed.
+
+    Latency is measured to the LAST chunk (the response as the caller saw it),
+    and the time to the FIRST chunk is reported separately as
+    ``time_to_first_token_ms`` - the two numbers a streaming call is judged by.
+
+    Attribute access falls through to the wrapped stream (``.response``,
+    provider helpers), and ``__iter__``/``__aiter__`` return ``self`` so early
+    ``break`` leaves no half-driven generator behind.
+    """
+
+    def __init__(
+        self,
+        stream: Any,
+        accumulator: StreamAccumulator,
+        on_finish: Callable[[Dict[str, Any], Optional[str]], None],
+    ) -> None:
+        self._stream = stream
+        self._accumulator = accumulator
+        self._on_finish = on_finish
+        self._done = False
+        self._first_chunk_t: Optional[float] = None
+        self._last_chunk_t: Optional[float] = None
+        self._sync_iter: Any = None
+        self._async_iter: Any = None
+
+    # -- bookkeeping ---------------------------------------------------------
+
+    def _observe(self, chunk: Any) -> None:
+        now = time.time()
+        if self._first_chunk_t is None:
+            self._first_chunk_t = now
+        self._last_chunk_t = now
+        try:
+            self._accumulator.feed(chunk)
+        except Exception:
+            # A malformed chunk must never break the caller's stream; it just
+            # goes uncounted in the trace.
+            pass
+
+    def _finish(self, error: Optional[str]) -> None:
+        if self._done:
+            return
+        self._done = True
+        try:
+            result = self._accumulator.result()
+        except Exception:
+            result = {}
+        start_t = result.pop("_start_t", None)
+        result["time_to_first_token_ms"] = (
+            int((self._first_chunk_t - start_t) * 1000) if self._first_chunk_t is not None and start_t is not None else None
+        )
+        # The response "ended" at its last chunk, not at whatever later moment the caller closed
+        # or dropped the stream - that is the latency the user experienced.
+        result["end_t"] = self._last_chunk_t if self._last_chunk_t is not None else time.time()
+        self._on_finish(result, error)
+
+    @property
+    def first_chunk_at(self) -> Optional[float]:
+        return self._first_chunk_t
+
+    # -- sync iteration ------------------------------------------------------
+
+    def __iter__(self) -> "TracedStream":
+        return self
+
+    def __next__(self) -> Any:
+        if self._sync_iter is None:
+            self._sync_iter = iter(self._stream)
+        try:
+            chunk = next(self._sync_iter)
+        except StopIteration:
+            self._finish(None)
+            raise
+        except BaseException as exc:
+            self._finish(str(exc))
+            raise
+        self._observe(chunk)
+        return chunk
+
+    # -- async iteration -----------------------------------------------------
+
+    def __aiter__(self) -> "TracedStream":
+        return self
+
+    async def __anext__(self) -> Any:
+        if self._async_iter is None:
+            self._async_iter = self._stream.__aiter__()
+        try:
+            chunk = await self._async_iter.__anext__()
+        except StopAsyncIteration:
+            self._finish(None)
+            raise
+        except BaseException as exc:
+            self._finish(str(exc))
+            raise
+        self._observe(chunk)
+        return chunk
+
+    # -- context managers / close --------------------------------------------
+
+    def __enter__(self) -> "TracedStream":
+        enter = getattr(self._stream, "__enter__", None)
+        if enter is not None:
+            enter()
+        return self
+
+    def __exit__(self, exc_type, exc_val, tb) -> Any:
+        exit_ = getattr(self._stream, "__exit__", None)
+        result = exit_(exc_type, exc_val, tb) if exit_ is not None else None
+        self._finish(str(exc_val) if exc_val else None)
+        return result
+
+    async def __aenter__(self) -> "TracedStream":
+        enter = getattr(self._stream, "__aenter__", None)
+        if enter is not None:
+            await enter()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, tb) -> Any:
+        exit_ = getattr(self._stream, "__aexit__", None)
+        result = await exit_(exc_type, exc_val, tb) if exit_ is not None else None
+        self._finish(str(exc_val) if exc_val else None)
+        return result
+
+    def close(self) -> None:
+        close = getattr(self._stream, "close", None)
+        try:
+            if close is not None:
+                close()
+        finally:
+            self._finish(None)
+
+    async def aclose(self) -> None:
+        # openai's AsyncStream spells its close as `async def close()`; httpx-style streams
+        # spell it `aclose()`. Await whichever one answers with an awaitable.
+        close = getattr(self._stream, "aclose", None) or getattr(self._stream, "close", None)
+        try:
+            if close is not None:
+                result = close()
+                if inspect.isawaitable(result):
+                    await result
+        finally:
+            self._finish(None)
+
+    def __getattr__(self, item: str) -> Any:
+        return getattr(self._stream, item)
+
+    def __del__(self) -> None:
+        # Best effort only: a stream the caller stopped reading and dropped still records the
+        # chunks it did see. Never raises - a destructor exception is unactionable noise.
+        try:
+            self._finish(None)
+        except Exception:
+            pass
+
+
+def trace_stream(
+    result: Any,
+    accumulator: StreamAccumulator,
+    on_finish: Callable[[Dict[str, Any], Optional[str]], None],
+) -> Any:
+    """
+    Wrap the value a patched ``create(..., stream=True)`` returned. A sync
+    client hands back the stream object directly; an async client hands back
+    a coroutine that resolves to it, so the wrapping is deferred until the
+    real stream exists - the caller's ``await`` is unchanged either way.
+    """
+    if asyncio.iscoroutine(result) or inspect.isawaitable(result):
+        return _await_and_wrap(result, accumulator, on_finish)
+    return TracedStream(result, accumulator, on_finish)
+
+
+async def _await_and_wrap(
+    awaitable: Any,
+    accumulator: StreamAccumulator,
+    on_finish: Callable[[Dict[str, Any], Optional[str]], None],
+) -> Any:
+    try:
+        stream = await awaitable
+    except Exception as exc:
+        on_finish({}, str(exc))
+        raise
+    return TracedStream(stream, accumulator, on_finish)

agentx/integrations/anthropic.py

@@ -13,6 +13,12 @@ Usage::
 
 Works with both ``anthropic.Anthropic`` and ``anthropic.AsyncAnthropic`` clients.
 
+Both streaming shapes are traced: the ``client.messages.stream(...)`` helper
+(a context manager with ``get_final_message()``) and the raw
+``messages.create(..., stream=True)`` event stream, which is wrapped in a
+transparent proxy that assembles the reply, tool-use blocks, and token usage
+from the events as the caller consumes them.
+
 Requires: ``pip install "agentx-python[anthropic]"``
 """
 from __future__ import annotations
@@ -22,7 +28,13 @@ import time
 from typing import Any, Dict, Optional, Tuple
 
 from agentx.tracing.tracer import Tracer, _safe_serialize
-from agentx.integrations._traced_call import capture_tool_definitions, call_and_trace, finish_llm_call
+from agentx.integrations._traced_call import (
+    StreamAccumulator,
+    capture_tool_definitions,
+    call_and_trace,
+    finish_llm_call,
+    trace_stream,
+)
 
 
 def _extract_output_text(response: Any) -> Optional[str]:
@@ -92,6 +104,85 @@ def _extract_usage_tokens(
     return input_tokens, output_tokens, cache_read, cache_creation
 
 
+class _MessageEventStreamAccumulator(StreamAccumulator):
+    """
+    Rebuild a ``Message`` from the raw ``create(stream=True)`` event sequence:
+    ``message_start`` carries the input-side usage, ``content_block_start`` opens
+    a text or tool_use block, ``content_block_delta`` appends ``text_delta`` /
+    ``input_json_delta`` fragments to it, ``message_delta`` carries the
+    output-token count. Token accounting mirrors ``_extract_usage_tokens``.
+    """
+
+    def __init__(self, start_t: float) -> None:
+        self._start_t = start_t
+        self._blocks: Dict[int, Dict[str, Any]] = {}
+        self._input_tokens: Optional[int] = None
+        self._output_tokens: Optional[int] = None
+        self._cache_read: Optional[int] = None
+        self._cache_write: Optional[int] = None
+        self._model: Optional[str] = None
+
+    def feed(self, event: Any) -> None:
+        event_type = getattr(event, "type", None)
+        if event_type == "message_start":
+            message = getattr(event, "message", None)
+            self._model = getattr(message, "model", None) or self._model
+            input_tokens, output_tokens, cache_read, cache_write = _extract_usage_tokens(getattr(message, "usage", None))
+            self._input_tokens = input_tokens
+            self._cache_read = cache_read
+            self._cache_write = cache_write
+            if output_tokens:
+                self._output_tokens = output_tokens
+        elif event_type == "content_block_start":
+            index = getattr(event, "index", 0) or 0
+            block = getattr(event, "content_block", None)
+            self._blocks[index] = {
+                "type": getattr(block, "type", None),
+                "name": getattr(block, "name", None),
+                "text": [getattr(block, "text", None) or ""] if getattr(block, "type", None) == "text" else [],
+                "json": [],
+            }
+        elif event_type == "content_block_delta":
+            index = getattr(event, "index", 0) or 0
+            delta = getattr(event, "delta", None)
+            entry = self._blocks.setdefault(index, {"type": None, "name": None, "text": [], "json": []})
+            delta_type = getattr(delta, "type", None)
+            if delta_type == "text_delta":
+                entry["type"] = entry["type"] or "text"
+                entry["text"].append(getattr(delta, "text", None) or "")
+            elif delta_type == "input_json_delta":
+                entry["type"] = entry["type"] or "tool_use"
+                entry["json"].append(getattr(delta, "partial_json", None) or "")
+        elif event_type == "message_delta":
+            usage = getattr(event, "usage", None)
+            output_tokens = getattr(usage, "output_tokens", None) if usage is not None else None
+            if output_tokens is not None:
+                self._output_tokens = output_tokens
+
+    def result(self) -> Dict[str, Any]:
+        texts = []
+        tool_calls = []
+        for _, block in sorted(self._blocks.items()):
+            if block["type"] == "text":
+                text = "".join(block["text"])
+                if text:
+                    texts.append(text)
+            elif block["type"] == "tool_use":
+                tool_calls.append(f"{block['name'] or 'unknown'}({''.join(block['json'])})")
+        output: Optional[str] = "\n".join(texts) if texts else None
+        if output is None and tool_calls:
+            output = "[tool call] " + ", ".join(tool_calls)
+        return {
+            "_start_t": self._start_t,
+            "output": output,
+            "model": self._model,
+            "input_tokens": self._input_tokens,
+            "output_tokens": self._output_tokens,
+            "cache_read_tokens": self._cache_read,
+            "cache_write_tokens": self._cache_write,
+        }
+
+
 def patch_anthropic_client(
     client: Any,
     tracer: Tracer,
@@ -140,6 +231,35 @@ def _patch_create(
 
         input_repr = _safe_serialize(input_messages)
 
+        if kwargs.get("stream"):
+            def on_stream_finish(collected: Dict[str, Any], error: Optional[str]) -> None:
+                finish_llm_call(
+                    tracer,
+                    name=name,
+                    framework="anthropic",
+                    metadata=metadata,
+                    call_metadata={"streaming": True, "timeToFirstTokenMs": collected.get("time_to_first_token_ms")},
+                    session_id=session_id,
+                    start_t=start_t,
+                    end_t=collected.get("end_t") or time.time(),
+                    input_repr=input_repr,
+                    output=collected.get("output"),
+                    model=collected.get("model") or model,
+                    input_tokens=collected.get("input_tokens"),
+                    output_tokens=collected.get("output_tokens"),
+                    cache_read_tokens=collected.get("cache_read_tokens"),
+                    cache_write_tokens=collected.get("cache_write_tokens"),
+                    error=error,
+                    tool_definitions=tool_definitions,
+                )
+
+            try:
+                result = original(*args, **kwargs)
+            except Exception as exc:
+                on_stream_finish({}, str(exc))
+                raise
+            return trace_stream(result, _MessageEventStreamAccumulator(start_t), on_stream_finish)
+
         def on_finish(response: Optional[Any], error: Optional[str]) -> None:
             end_t = time.time()
             output = None
@@ -237,36 +357,47 @@ def _patch_stream(
             )
 
         class _TracedStream:
-            """Thin wrapper that records timing when the stream context exits."""
+            """
+            Thin wrapper that records the final message when the stream context exits.
+            ``ctx`` is the SDK's stream *manager*; the ``MessageStream`` it yields on enter is
+            what carries ``get_final_message()``, and it must be read BEFORE the manager's exit
+            closes it - reading it off the manager after close silently yielded no output.
+            """
+
+            _inner: Any = None
 
             def __enter__(self_inner):
-                return ctx.__enter__()
+                self_inner._inner = ctx.__enter__()
+                return self_inner._inner
 
             def __exit__(self_inner, exc_type, exc_val, tb):
-                result = ctx.__exit__(exc_type, exc_val, tb)
                 end_t = time.time()
                 error = str(exc_val) if exc_val else None
                 final_message = None
-                try:
-                    final_message = ctx.get_final_message()
-                except Exception:
-                    pass
+                if error is None and self_inner._inner is not None:
+                    try:
+                        final_message = self_inner._inner.get_final_message()
+                    except Exception:
+                        pass
+                result = ctx.__exit__(exc_type, exc_val, tb)
                 build_and_send(end_t, error, final_message)
                 return result
 
             async def __aenter__(self_inner):
-                return await ctx.__aenter__()
+                self_inner._inner = await ctx.__aenter__()
+                return self_inner._inner
 
             async def __aexit__(self_inner, exc_type, exc_val, tb):
-                result = await ctx.__aexit__(exc_type, exc_val, tb)
                 end_t = time.time()
                 error = str(exc_val) if exc_val else None
                 final_message = None
-                try:
-                    raw = ctx.get_final_message()
-                    final_message = await raw if inspect.isawaitable(raw) else raw
-                except Exception:
-                    pass
+                if error is None and self_inner._inner is not None:
+                    try:
+                        raw = self_inner._inner.get_final_message()
+                        final_message = await raw if inspect.isawaitable(raw) else raw
+                    except Exception:
+                        pass
+                result = await ctx.__aexit__(exc_type, exc_val, tb)
                 build_and_send(end_t, error, final_message)
                 return result
 

agentx/integrations/openai.py

@@ -17,8 +17,11 @@ Usage::
 
 Works with both ``openai.OpenAI`` and ``openai.AsyncOpenAI`` clients.
 
-Streaming calls (``stream=True``) are passed through untouched and are not
-currently traced - see ``patch_openai_client``'s docstring.
+Streaming calls (``stream=True``) are traced too: the returned stream is
+wrapped in a transparent proxy that assembles the reply from the chunks as the
+caller consumes them, so the trace carries the full text, tool calls, and
+(with ``stream_options={"include_usage": True}``) token usage, plus the time
+to first token.
 
 Requires: ``pip install "agentx-python[openai]"``
 """
@@ -28,7 +31,13 @@ import time
 from typing import Any, Dict, Optional, Tuple
 
 from agentx.tracing.tracer import Tracer, _safe_serialize
-from agentx.integrations._traced_call import capture_tool_definitions, call_and_trace, finish_llm_call
+from agentx.integrations._traced_call import (
+    StreamAccumulator,
+    capture_tool_definitions,
+    call_and_trace,
+    finish_llm_call,
+    trace_stream,
+)
 
 
 def _extract_output_text(response: Any) -> Optional[str]:
@@ -75,6 +84,68 @@ def _extract_usage_tokens(usage: Any) -> Tuple[Optional[int], Optional[int], Opt
     return getattr(usage, "prompt_tokens", None), getattr(usage, "completion_tokens", None), cached_tokens
 
 
+class _ChatCompletionStreamAccumulator(StreamAccumulator):
+    """
+    Rebuild a ``ChatCompletion``-shaped result from ``ChatCompletionChunk``s:
+    text deltas concatenate per choice, tool-call deltas merge by index (name
+    arrives once, arguments arrive as fragments), and the ``usage`` block -
+    present only on the final chunk, and only when the caller asked for it
+    with ``stream_options={"include_usage": True}`` - is kept when it appears.
+    """
+
+    def __init__(self, start_t: float) -> None:
+        self._start_t = start_t
+        self._texts: Dict[int, list] = {}
+        self._tool_calls: Dict[int, Dict[str, Any]] = {}
+        self._usage: Any = None
+        self._model: Optional[str] = None
+
+    def feed(self, chunk: Any) -> None:
+        usage = getattr(chunk, "usage", None)
+        if usage is not None:
+            self._usage = usage
+        model = getattr(chunk, "model", None)
+        if model and not self._model:
+            self._model = model
+        for choice in getattr(chunk, "choices", None) or []:
+            index = getattr(choice, "index", 0) or 0
+            delta = getattr(choice, "delta", None)
+            if delta is None:
+                continue
+            content = getattr(delta, "content", None)
+            if content:
+                self._texts.setdefault(index, []).append(content)
+            for tc in getattr(delta, "tool_calls", None) or []:
+                key = getattr(tc, "index", 0) or 0
+                entry = self._tool_calls.setdefault(key, {"name": None, "arguments": []})
+                fn = getattr(tc, "function", None)
+                fn_name = getattr(fn, "name", None) if fn is not None else None
+                fn_args = getattr(fn, "arguments", None) if fn is not None else None
+                if fn_name:
+                    entry["name"] = fn_name
+                if fn_args:
+                    entry["arguments"].append(fn_args)
+
+    def result(self) -> Dict[str, Any]:
+        texts = ["".join(parts) for _, parts in sorted(self._texts.items())]
+        output: Optional[str] = "\n".join(t for t in texts if t) or None
+        if output is None and self._tool_calls:
+            described = [
+                f"{entry['name'] or 'unknown'}({''.join(entry['arguments'])})"
+                for _, entry in sorted(self._tool_calls.items())
+            ]
+            output = "[tool call] " + ", ".join(described)
+        input_tokens, output_tokens, cache_read_tokens = _extract_usage_tokens(self._usage)
+        return {
+            "_start_t": self._start_t,
+            "output": output,
+            "model": self._model,
+            "input_tokens": input_tokens,
+            "output_tokens": output_tokens,
+            "cache_read_tokens": cache_read_tokens,
+        }
+
+
 def patch_openai_client(
     client: Any,
     tracer: Tracer,
@@ -92,12 +163,14 @@ def patch_openai_client(
     client's ``create()`` returns a coroutine, which is detected and awaited
     before the trace is built.
 
-    Calls made with ``stream=True`` are passed through untouched and are not
-    traced by this function: safely wrapping a (sync or async) chunk
-    iterator without disrupting the caller's own consumption of it needs
-    different handling than a single request/response call, so it's left
-    unpatched rather than risking a partially-consumed or double-consumed
-    stream for the caller.
+    Calls made with ``stream=True`` return a transparent proxy over the
+    provider's stream (see ``_traced_call.TracedStream``): iteration, ``with``,
+    ``close()`` and attribute access all pass through to the real stream, and
+    the trace is built from the chunks the caller actually consumed - text and
+    tool calls assembled from the deltas, token usage from the final chunk
+    when ``stream_options={"include_usage": True}`` was requested (OpenAI omits
+    usage from streams otherwise), latency to the last chunk, and the time to
+    first token in the trace metadata.
     """
     chat = getattr(client, "chat", None)
     completions = getattr(chat, "completions", None) if chat is not None else None
@@ -123,17 +196,41 @@ def _patch_chat_completions_create(
         return  # already patched
 
     def patched_create(*args, **kwargs):
-        if kwargs.get("stream"):
-            # Not traced - see patch_openai_client's docstring. Passed
-            # through completely untouched, sync or async.
-            return original(*args, **kwargs)
-
         start_t = time.time()
         input_messages = kwargs.get("messages") or (args[0] if args else None)
         model = kwargs.get("model")
         input_repr = _safe_serialize(input_messages)
         tool_definitions = capture_tool_definitions(kwargs.get("tools"))
 
+        if kwargs.get("stream"):
+            def on_stream_finish(collected: Dict[str, Any], error: Optional[str]) -> None:
+                ttft = collected.get("time_to_first_token_ms")
+                finish_llm_call(
+                    tracer,
+                    name=name,
+                    framework=framework,
+                    metadata=metadata,
+                    call_metadata={"streaming": True, "timeToFirstTokenMs": ttft},
+                    session_id=session_id,
+                    start_t=start_t,
+                    end_t=collected.get("end_t") or time.time(),
+                    input_repr=input_repr,
+                    output=collected.get("output"),
+                    model=collected.get("model") or model,
+                    input_tokens=collected.get("input_tokens"),
+                    output_tokens=collected.get("output_tokens"),
+                    cache_read_tokens=collected.get("cache_read_tokens"),
+                    error=error,
+                    tool_definitions=tool_definitions,
+                )
+
+            try:
+                result = original(*args, **kwargs)
+            except Exception as exc:
+                on_stream_finish({}, str(exc))
+                raise
+            return trace_stream(result, _ChatCompletionStreamAccumulator(start_t), on_stream_finish)
+
         def on_finish(response: Optional[Any], error: Optional[str]) -> None:
             end_t = time.time()
             output = None

agentx/monitor/__init__.py

@@ -12,12 +12,16 @@ from agentx.monitor.patterns import MonitorPatternBuilder, MonitorPatternClient
 from agentx.monitor.profile import MonitorProfileClient
 from agentx.monitor.review_queue import ReviewQueueClient, ReviewQueueItem
 from agentx.monitor.rules import MonitorRule, MonitorRulesClient
+from agentx.monitor.alert_rules import AlertEvent, AlertRule, AlertRulesClient
 from agentx.monitor.scorers import AgentXScorersError, ScorersClient
 from agentx.monitor.scorer_groups import AgentXScorerGroupsError, ScorerGroup, ScorerGroupsClient
 from agentx.monitor.sessions import MonitorSessionClient
 from agentx.monitor.signals import MonitorSignalClient
 
 __all__ = [
+    "AlertEvent",
+    "AlertRule",
+    "AlertRulesClient",
     "AgentXImprovementGroupsError",
     "AgentXJudgeScorersError",
     "AgentXMonitorError",

agentx/monitor/alert_rules.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from agentx.monitor.client import MonitorClient
+
+ALERT_METRICS = ("failureRate", "toolFailureRate", "p95LatencyMs", "estimatedCostUsd", "judgeFailures", "traceCount")
+ALERT_CHANNEL_KINDS = ("slack", "teams", "pagerduty", "email", "webhook")
+
+# snake_case kwargs -> wire keys. The engine reads camelCase only (the wire convention); an
+# unknown snake_case key would be silently ignored, so update() refuses it instead.
+_ALIASES = {
+    "window_minutes": "windowMinutes",
+    "agent_id": "agentId",
+    "cooldown_minutes": "cooldownMinutes",
+}
+
+
+class AlertRule(dict):
+    """Wire object for one KPI alert rule (dict subclass so unknown fields round-trip)."""
+
+    @property
+    def id(self) -> str:
+        return self["_id"]
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.get("enabled"))
+
+    @property
+    def state(self) -> str:
+        """``"ok"`` or ``"firing"``."""
+        return str(self.get("state") or "ok")
+
+    @property
+    def last_value(self) -> Optional[float]:
+        return self.get("lastValue")
+
+    @property
+    def fired_count(self) -> int:
+        return int(self.get("firedCount") or 0)
+
+
+class AlertEvent(dict):
+    """One row of a rule's notification history: ``kind`` is ``triggered`` / ``repeat`` /
+    ``resolved`` / ``test``; ``deliveries`` lists each channel's outcome."""
+
+    @property
+    def kind(self) -> str:
+        return str(self.get("kind"))
+
+    @property
+    def delivered(self) -> bool:
+        deliveries = self.get("deliveries") or []
+        return bool(deliveries) and all(bool(d.get("ok")) for d in deliveries)
+
+
+def slack(url: str) -> Dict[str, str]:
+    """A Slack incoming-webhook channel."""
+    return {"kind": "slack", "target": url}
+
+
+def teams(url: str) -> Dict[str, str]:
+    """A Microsoft Teams incoming-webhook (or Workflows) channel."""
+    return {"kind": "teams", "target": url}
+
+
+def pagerduty(routing_key: str) -> Dict[str, str]:
+    """A PagerDuty Events API v2 integration - ``routing_key`` is the integration key."""
+    return {"kind": "pagerduty", "target": routing_key}
+
+
+def email(address: str) -> Dict[str, str]:
+    """An email recipient (needs a mailer configured on the engine)."""
+    return {"kind": "email", "target": address}
+
+
+def webhook(url: str) -> Dict[str, str]:
+    """A generic JSON webhook receiving the full structured notification."""
+    return {"kind": "webhook", "target": url}
+
+
+class AlertRulesClient:
+    """Surfaced as ``client.monitor.alert_rules``: KPI alert rules.
+
+    An alert rule watches an AGGREGATE over a sliding window - one of
+    ``failureRate``, ``toolFailureRate``, ``p95LatencyMs``, ``estimatedCostUsd``,
+    ``judgeFailures``, ``traceCount`` - and pages typed channels (Slack, Teams,
+    PagerDuty, email, generic webhook) when it crosses a threshold. The engine
+    evaluates every enabled rule once a minute with an Alertmanager-style
+    lifecycle: one ``triggered`` notification when the rule starts breaching, a
+    ``repeat`` every ``cooldown_minutes`` while it keeps breaching, and a
+    ``resolved`` notification when it recovers. This is distinct from a scorer's
+    per-verdict alert threshold and from an automation rule's per-trace routing.
+
+    Example::
+
+        from agentx.monitor.alert_rules import slack, pagerduty
+
+        rule = client.monitor.alert_rules.create(
+            "Failure rate above 10%",
+            metric="failureRate", operator="gt", threshold=0.10, window_minutes=15,
+            severity="high",
+            channels=[slack("https://hooks.slack.com/services/..."), pagerduty("R0123...")],
+        )
+        client.monitor.alert_rules.test(rule.id)   # sends a TEST page to every channel
+    """
+
+    def __init__(self, client: "MonitorClient"):
+        self._client = client
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> Any:
+        return self._client._request(method, path, base=self._client._api_root(), **kwargs)
+
+    def list(self) -> List[AlertRule]:
+        data = self._request("GET", "/agent-monitoring/alert-rules")
+        return [AlertRule(r) for r in data.get("rules", [])]
+
+    def get(self, rule_id: str) -> AlertRule:
+        data = self._request("GET", f"/agent-monitoring/alert-rules/{rule_id}")
+        return AlertRule(data.get("rule", data))
+
+    def create(
+        self,
+        name: str,
+        *,
+        metric: str,
+        operator: str,
+        threshold: float,
+        window_minutes: int,
+        channels: List[Dict[str, str]],
+        agent_id: Optional[str] = None,
+        severity: str = "high",
+        cooldown_minutes: int = 60,
+        enabled: bool = True,
+    ) -> AlertRule:
+        """Create a rule. ``operator`` is ``"gt"`` (above) or ``"lt"`` (below); rates are
+        fractions (``0.10`` = 10%), latency is milliseconds, cost is USD. ``channels`` takes the
+        dicts the module-level helpers build (``slack(url)``, ``pagerduty(key)``, ...)."""
+        if metric not in ALERT_METRICS:
+            raise ValueError(f"metric must be one of {ALERT_METRICS}, got {metric!r}")
+        if operator not in ("gt", "lt"):
+            raise ValueError(f"operator must be 'gt' or 'lt', got {operator!r}")
+        for channel in channels:
+            if channel.get("kind") not in ALERT_CHANNEL_KINDS:
+                raise ValueError(f"channel kind must be one of {ALERT_CHANNEL_KINDS}, got {channel.get('kind')!r}")
+        payload: Dict[str, Any] = {
+            "name": name,
+            "metric": metric,
+            "operator": operator,
+            "threshold": threshold,
+            "windowMinutes": window_minutes,
+            "channels": channels,
+            "severity": severity,
+            "cooldownMinutes": cooldown_minutes,
+            "enabled": enabled,
+        }
+        if agent_id is not None:
+            payload["agentId"] = agent_id
+        # Server-side write: a timeout retry would create a duplicate rule that pages twice on
+        # every incident - no transport retry (same posture as rules.create).
+        data = self._request("POST", "/agent-monitoring/alert-rules", json=payload, retry=False)
+        return AlertRule(data.get("rule", data))
+
+    def update(self, rule_id: str, **fields: Any) -> AlertRule:
+        """Sparse update; snake_case kwargs are mapped to the wire. Changing the metric,
+        operator, threshold, window, or agent resets the rule's firing state."""
+        payload: Dict[str, Any] = {}
+        for key, value in fields.items():
+            wire_key = _ALIASES.get(key, key)
+            if "_" in wire_key:
+                raise ValueError(
+                    f"Unknown alert rule field {key!r} - the engine reads camelCase keys and would "
+                    "silently ignore this (see AlertRulesClient.create for the field names)."
+                )
+            payload[wire_key] = value
+        data = self._request("PUT", f"/agent-monitoring/alert-rules/{rule_id}", json=payload)
+        return AlertRule(data.get("rule", data))
+
+    def delete(self, rule_id: str) -> None:
+        """Deletes the rule and its history. A PagerDuty incident the rule opened is not
+        resolved by this - close it in PagerDuty."""
+        self._request("DELETE", f"/agent-monitoring/alert-rules/{rule_id}", retry=False)
+
+    def events(self, rule_id: str, limit: int = 50) -> List[AlertEvent]:
+        """The rule's notification history, newest first, with per-channel delivery results."""
+        data = self._request("GET", f"/agent-monitoring/alert-rules/{rule_id}/events", params={"limit": limit})
+        return [AlertEvent(e) for e in data.get("events", [])]
+
+    def test(self, rule_id: str) -> AlertEvent:
+        """Send a TEST notification to the rule's channels with the metric's live value and
+        return the recorded event - ``event.delivered`` says whether every channel accepted it.
+        Never changes the rule's firing state."""
+        data = self._request("POST", f"/agent-monitoring/alert-rules/{rule_id}/test", json={}, retry=False)
+        return AlertEvent(data.get("event", data))
+
+    def preview(self, metric: str, window_minutes: int, agent_id: Optional[str] = None) -> Dict[str, Any]:
+        """What ``metric`` reads right now over the last ``window_minutes`` - the same
+        computation the sweep runs. Returns ``{"value": float | None, "valueLabel": str, ...}``;
+        ``value`` is ``None`` when the window has no data for a rate metric."""
+        payload: Dict[str, Any] = {"metric": metric, "windowMinutes": window_minutes}
+        if agent_id is not None:
+            payload["agentId"] = agent_id
+        return self._request("POST", "/agent-monitoring/alert-rules/preview", json=payload)
+
+    def run_sweep(self) -> Dict[str, Any]:
+        """Evaluate this project's rules now instead of waiting for the next minute tick."""
+        return self._request("POST", "/agent-monitoring/alert-rules/sweep/run", json={}, retry=False)

agentx/monitor/client.py

@@ -139,6 +139,11 @@ class MonitorClient:
 
         # Automation rules: route matching traffic into review / a dataset / a webhook.
         self.rules = MonitorRulesClient(self)
+        from agentx.monitor.alert_rules import AlertRulesClient
+
+        # KPI alert rules: threshold pages (Slack/Teams/PagerDuty/email/webhook) on failure
+        # rate, p95 latency, spend, judge failures, and traffic volume over a window.
+        self.alert_rules = AlertRulesClient(self)
         from agentx.monitor.scorers import ScorersClient
         # Scorers-catalog administration as code: template enable/disable, code/external scorer
         # CRUD and dry runs - full parity with the dashboard's Scorers page (P1.3).

agentx/tracing/tracer.py

@@ -297,6 +297,7 @@ class _TraceSpan:
         output_tokens: Optional[int] = None,
         cache_read_tokens: Optional[int] = None,
         cache_write_tokens: Optional[int] = None,
+        metadata: Optional[Dict[str, Any]] = None,
     ) -> None:
         """Record one LLM-call child span (e.g. one patched Anthropic call) under this span -
         name left unset so _merge_child_run auto-numbers it "LLM Call N". ``framework`` lets the
@@ -315,6 +316,9 @@ class _TraceSpan:
                 "outputTokenSize": output_tokens,
                 "cacheReadTokenSize": cache_read_tokens,
                 "cacheWriteTokenSize": cache_write_tokens,
+                # Per-call facts that belong on the child row (a streamed call's time to first
+                # token), not on the parent trace's metadata.
+                "metadata": metadata,
             }],
             input=input,
             output=output,
@@ -479,6 +483,7 @@ class _TraceSpan:
                     output_tokens=step.get("outputTokenSize"),
                     cache_read_tokens=step.get("cacheReadTokenSize"),
                     cache_write_tokens=step.get("cacheWriteTokenSize"),
+                    metadata=step.get("metadata") or None,
                     # Stated, so a step named anything other than "LLM Call N" still classifies -
                     # the backend's name regex was the only thing holding this together. Steps
                     # may state their own kind (crewai.py's task steps carry "agent"); the

agentx/version.py

@@ -1,7 +1,7 @@
-VERSION = "0.8.26"
+VERSION = "0.8.27"
 
 # The AgentX-trace-eval release this SDK version is tested against - what `agentx-trace-eval`
 # installs and converges to (see agentx/cli.py). Bump together with VERSION when releasing, so
 # every published SDK names a known-good engine+dashboard pair. Users can override with
 # AGENTX_TRACE_EVAL_VERSION=<tag|latest>.
-ENGINE_VERSION = "v0.3.29"
+ENGINE_VERSION = "v0.3.30"

{agentx_python-0.8.26.dist-info → agentx_python-0.8.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.26
+Version: 0.8.27
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.8.26.dist-info → agentx_python-0.8.27.dist-info}/RECORD

@@ -10,7 +10,7 @@ agentx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 agentx/testing.py,sha256=0shZEid_vhJgBegpO_loUq6APYDxQrUgD3jvcRdIDV8,7372
 agentx/traces.py,sha256=Jh07n3GLS1SGRsDoLDTUaAbuMEdvjqxCCMA9gUcSZ0M,2331
 agentx/util.py,sha256=ivCuFQ5AGUw9fR9fJy5HrSp3lLwwngsDwppnvyZpq9E,1471
-agentx/version.py,sha256=mHuhKZSPSp8Xp5Pq7q93wgdcMtBUM_OBt-nWG4Qoh6g,366
+agentx/version.py,sha256=KdSKPJMRW7kj12djMaC4Y9MJOtmKcvAm9zC6IV3Qeig,366
 agentx/evaluations/__init__.py,sha256=Erv7RGFlRxGTG4rVb2uHhCqLLEX_iAWKqkZKzK6CumE,262
 agentx/evaluations/_term.py,sha256=WFpiNzdgDBeJJ-Gg-6X7TwwxllobuE4OqFUTuDQvS3Y,2529
 agentx/evaluations/client.py,sha256=7c35tS8zM3fuvFfVDQk5etTTEFiIykmPjuJxkuFyFTY,38013
@@ -28,8 +28,8 @@ agentx/evaluations/adapters/http_endpoint.py,sha256=-Gika9lBcXjtbnyB-uNR6nYEVJhW
 agentx/evaluations/adapters/precomputed.py,sha256=vxnQELmpiU2NkQATfiVjG7v1ZmmkrEA70Y11ijfYtHw,1208
 agentx/evaluations/adapters/raw.py,sha256=FkDq_mdf21-xt5EHnGyIGcS6VZxkEkFcmM9VTd-T5sA,1130
 agentx/integrations/__init__.py,sha256=p-YHLIudYxQj1bBj77NiVPsP0VQHemX5E59phkI3Y-Q,711
-agentx/integrations/_traced_call.py,sha256=OuFmSLw5vjI54gpdte7-zOt7gld7faQmvhuoh_3OPlo,6861
-agentx/integrations/anthropic.py,sha256=z6o9cC9rBzxxbF1UBiMr-kjEgDUO-BCuPOFoqGN3qF4,10451
+agentx/integrations/_traced_call.py,sha256=I8t37HUVp-VJYcHo26zkJs80A7S8ajh7F9JQ_IRfNCU,14740
+agentx/integrations/anthropic.py,sha256=y-6A32X8CLrwttjSiJuDWunQmi9QuzMYhWHMuutfRJY,16655
 agentx/integrations/autogen.py,sha256=V1kvTjxQqOG9KsYd4dztaRYH7zTknW0_fqzKjIQG5dk,9507
 agentx/integrations/crewai.py,sha256=SCObRsrS40P8jOGoI_T5oW4IYuoBW5WjBNuc7zUBU70,14346
 agentx/integrations/databricks.py,sha256=vKXnur-LWMTzctFuIKte2N826sydXVlpi1qFXo2dfio,18044
@@ -40,12 +40,13 @@ agentx/integrations/litellm.py,sha256=eW8iCaNq0feRbZma8iOvmJ28WiAO7ZaRmlpPjHAtcv
 agentx/integrations/llamaindex.py,sha256=ruzMiC2TC3Xhub4xoXKPHjsvzV1sQ3QqhjkAdMxUUi4,18001
 agentx/integrations/moveworks.py,sha256=IyBswLE5LwMSp1VbIvG5izYV5BvXmv4atZelnrDn8SE,25670
 agentx/integrations/nvidia_nim.py,sha256=bvS6ebEnQZT1DVlIjQQ2RKLI4uPPTdlMKINgDwj0Z-o,2961
-agentx/integrations/openai.py,sha256=edJL-HXTO7w-vvIZGQChMu0AnPCY5z0duplIi8z2gXE,6936
+agentx/integrations/openai.py,sha256=0DMKmbQ1Rdd-gIaJuGJ-JEcvMnQ6620d0EQcZrNC46Q,11249
 agentx/integrations/openai_agents.py,sha256=KSOQ55eRPFxxkqm7p4Dsz6HcJ_9uUEwlyywPI8dwbvY,14852
-agentx/monitor/__init__.py,sha256=Rkk-Xv1ymXwDFbx8fetmcOzXfSU1bvsazI_-x5Ati5g,1715
+agentx/monitor/__init__.py,sha256=37zUS2j8FZO98TXdrzE42td9kRIWhypm3yqUR5sBVeg,1853
 agentx/monitor/_transport.py,sha256=ge0pmdvFdUtCJRkTLvAgjuTQVA8YOCYGQNoyRuselRk,2077
 agentx/monitor/agents.py,sha256=KhsHVhkqa6V0e0lPcuo04fh44_qSaJOaGcD6U6qXSFo,1622
-agentx/monitor/client.py,sha256=JoKp3AmFtQ4UDOsNH5nWEkTjvQtBrrihObGaPVMtBYo,27270
+agentx/monitor/alert_rules.py,sha256=UgK6Cnrcz9F9p1QU-eAg1NsjrtfMoA-dwX8AWKwEcVc,8990
+agentx/monitor/client.py,sha256=oZctmh77Ki5WnZI6vQaj_tDlmdP6XtkDYt3MKhnvGsU,27563
 agentx/monitor/improvement_groups.py,sha256=MrXexaBLPNzZHrqZjiCPfmwa-DSr-RHuFdP73SKrMJM,6143
 agentx/monitor/judge_scorers.py,sha256=Xyoafo9ljeq5RpLe5HxvsL0Y0FebFQ-2CxYz8-E1Wio,21025
 agentx/monitor/models.py,sha256=mNBQDXdVthGYHUyEeRgpFdKVOC9uMtdUgBlLfep1E14,9298
@@ -67,10 +68,10 @@ agentx/tracing/ci_types.py,sha256=b-W2LowRhNBVUdaoMPgd4bnKQy9AxhfiikftRwGPAqU,17
 agentx/tracing/eval_scope.py,sha256=ElMbPxpqpQBVuaUnHNlw9RIQu71oyOpd0yR55bDH8eI,2144
 agentx/tracing/framework_detect.py,sha256=uV4O7Th-4_2UkdooAyaWCOIA0jwLJbblFQ_ucFDjeJM,2638
 agentx/tracing/ingest_client.py,sha256=teukTPckFWPg5RhpbGbVDBejBzPIPXUO3_DJcuVh2uc,22660
-agentx/tracing/tracer.py,sha256=NS0bs5HbDDK3l0Z95KVKV3xCNJT7LINS0EoPFM-wzIk,68543
-agentx_python-0.8.26.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
-agentx_python-0.8.26.dist-info/METADATA,sha256=JH0SY8e_dBGzmK92xd_Cas62qpPweH0BoBLbDmE7xPA,22986
-agentx_python-0.8.26.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-agentx_python-0.8.26.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
-agentx_python-0.8.26.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
-agentx_python-0.8.26.dist-info/RECORD,,
+agentx/tracing/tracer.py,sha256=a9BALnb7UyyOUMAIZW6z-4QkevsO3TsBkY8GiJUEnS4,68848
+agentx_python-0.8.27.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
+agentx_python-0.8.27.dist-info/METADATA,sha256=j6fxEPZbrq1VwHmUYEUGAvRawZH52BlTrogVjrYCy-0,22986
+agentx_python-0.8.27.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+agentx_python-0.8.27.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
+agentx_python-0.8.27.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
+agentx_python-0.8.27.dist-info/RECORD,,