agentx-python 0.6.23__tar.gz → 0.6.25__tar.gz

{agentx_python-0.6.23 → agentx_python-0.6.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.23
+Version: 0.6.25
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx/evaluations/datasets.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import csv
 import logging
 from pathlib import Path
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union
 
 from agentx.evaluations.models import Dataset
 
@@ -88,6 +88,7 @@ class DatasetBuilder:
         smoke_test_guidance: Optional[str] = None,
         expected_tools: Optional[List[str]] = None,
         trajectory_match_mode: str = "strict",
+        expected_retrieval_context: Optional[Union[str, List[str]]] = None,
     ) -> "DatasetBuilder":
         """Add a case. `judge_guideline` is optional grading guidance specific to this question.
 
@@ -99,6 +100,13 @@ class DatasetBuilder:
         any order), "superset" (all expected present, extras allowed), or "subset" (no unexpected
         calls, missing allowed).
 
+        `expected_retrieval_context` (string or list of chunk strings) declares what a correct
+        retriever should have fetched for this case. When the run's result carries actual
+        retrieved context (a `retrieval_context` return value, or a linked trace with retrieval
+        spans), the engine compares the two with token-level Jaccard similarity and reports a
+        deterministic "Context match (jaccard)" scorer row (0-1) - a cheap retriever regression
+        check with no LLM judge call.
+
         `smoke_test_count`, when set (1-10), asks this question that many extra ways each
         evaluation run, LLM-paraphrased server-side, to catch agents that are brittle to phrasing
         rather than genuinely wrong. `smoke_test_guidance` optionally steers what kind of variants
@@ -123,6 +131,8 @@ class DatasetBuilder:
                 main["smokeTest"]["guidance"] = smoke_test_guidance
         if expected_tools:
             main["expectedTrajectory"] = {"tools": expected_tools, "mode": trajectory_match_mode}
+        if expected_retrieval_context:
+            main["expectedRetrievalContext"] = expected_retrieval_context
         self._payload["questions"].append(
             {
                 "main_question": main,

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx/evaluations/models.py

@@ -346,6 +346,11 @@ class EvaluationResult(BaseModel):
     # the dashboard's "Message Trace Details -> Execution Timeline" render the full execution
     # trace for this case, not just the lightweight observable_trace events above.
     trace_id: Optional[str] = Field(default=None, alias="traceId")
+    # What the agent actually retrieved for THIS case (a string, or list of chunk strings) - the
+    # {context} the RAG metric pack's judges grade faithfulness/relevancy against. Overrides the
+    # question's static retrievalContext; when absent, the engine falls back to the linked
+    # trace's recorded retrieval spans (trace_id above), then the case's pinned context.
+    retrieval_context: Optional[Any] = Field(default=None, alias="retrievalContext")
     # Smoke test: set by execute() from the originating EvaluationCase, not something you need to
     # set yourself when returning a plain str/dict from your callable.
     is_smoke_test_variant: Optional[bool] = Field(default=None, alias="isSmokeTestVariant")

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx/evaluations/results.py

@@ -37,6 +37,7 @@ def normalize_result(
     output: Optional[dict] = None
     trace = None
     trace_id: Optional[str] = None
+    retrieval_context = None
     metadata: Optional[dict] = None
     error: Optional[ResultError] = None
     input_tokens: Optional[int] = None
@@ -55,6 +56,7 @@ def normalize_result(
         trace = build_trace(raw.get("trace") or raw.get("observable_trace"))
         trace_id_raw = raw.get("trace_id") or raw.get("traceId")
         trace_id = str(trace_id_raw) if trace_id_raw else None
+        retrieval_context = raw.get("retrieval_context") or raw.get("retrievalContext")
         meta_raw = raw.get("metadata")
         if isinstance(meta_raw, dict):
             metadata = redact_dict(meta_raw)
@@ -108,6 +110,7 @@ def normalize_result(
         ),
         metadata=metadata,
         traceId=trace_id,
+        retrievalContext=retrieval_context,
         isSmokeTestVariant=case.is_smoke_test_variant,
         smokeTestVariantText=case.smoke_test_variant_text,
     )

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx/integrations/_traced_call.py

@@ -13,9 +13,10 @@ from __future__ import annotations
 
 import asyncio
 import inspect
+import json
 from typing import Any, Callable, Dict, Optional
 
-from agentx.tracing.tracer import Tracer
+from agentx.tracing.tracer import Tracer, _safe_serialize
 
 
 def call_and_trace(
@@ -62,6 +63,32 @@ async def _await_and_finish(
     return response
 
 
+# The engine's unregistered-tool surfacing (Tools & MCPs -> Unregistered) reads a trace's
+# metadata "tools" key to show the REAL definition instead of one inferred from observed
+# arguments (see AgentX-trace-eval's toolSchemas.ts draftFromMetadata). Raw-client patches see
+# the request's tools=[...] right in kwargs, so capture it - capped so a huge toolbox never
+# blows up the trace's metadata budget.
+_MAX_TOOL_DEFINITIONS = 20
+_MAX_TOOL_DEFINITIONS_BYTES = 12_000
+
+
+def capture_tool_definitions(tools: Any) -> Optional[list]:
+    """Return a metadata-ready copy of a request's ``tools=[...]`` list, or None."""
+    if not isinstance(tools, list) or not tools:
+        return None
+    # A plain JSON round-trip preserves nested schema objects exactly (default=str catches the
+    # odd SDK object inside); _safe_serialize would repr-stringify nested dicts, turning a
+    # parameters schema into an unusable string.
+    try:
+        text = json.dumps(tools[:_MAX_TOOL_DEFINITIONS], default=str)
+        if len(text) > _MAX_TOOL_DEFINITIONS_BYTES:
+            return None
+        serialized = json.loads(text)
+    except (TypeError, ValueError):
+        return None
+    return serialized if isinstance(serialized, list) else None
+
+
 def finish_llm_call(
     tracer: Tracer,
     *,
@@ -79,6 +106,7 @@ def finish_llm_call(
     error: Optional[str],
     cache_read_tokens: Optional[int] = None,
     cache_write_tokens: Optional[int] = None,
+    tool_definitions: Optional[list] = None,
 ) -> None:
     """
     Close out one raw-client LLM call - shared by the ``on_finish``/exit
@@ -96,8 +124,16 @@ def finish_llm_call(
     """
     latency_ms = int((end_t - start_t) * 1000)
 
+    if tool_definitions:
+        metadata = {**(metadata or {}), "tools": tool_definitions}
+
     active_span = tracer.current_span
     if active_span is not None:
+        # The definitions describe the whole call's toolbox - attach them to the enclosing
+        # span's metadata (first capture wins) so the ROOT trace carries them for the
+        # unregistered-tool listing, same as the standalone-trace path below.
+        if tool_definitions and not (active_span._metadata or {}).get("tools"):
+            active_span._metadata = {**(active_span._metadata or {}), "tools": tool_definitions}
         if error is not None:
             active_span.set_error(error)
         active_span._record_llm_call(

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx/integrations/anthropic.py

@@ -22,7 +22,7 @@ import time
 from typing import Any, Dict, Optional, Tuple
 
 from agentx.tracing.tracer import Tracer, _safe_serialize
-from agentx.integrations._traced_call import call_and_trace, finish_llm_call
+from agentx.integrations._traced_call import capture_tool_definitions, call_and_trace, finish_llm_call
 
 
 def _extract_output_text(response: Any) -> Optional[str]:
@@ -136,6 +136,7 @@ def _patch_create(
             kwargs.get("system"),
         )
         model = kwargs.get("model")
+        tool_definitions = capture_tool_definitions(kwargs.get("tools"))
 
         input_repr = _safe_serialize(input_messages)
 
@@ -171,6 +172,7 @@ def _patch_create(
                 cache_read_tokens=cache_read_tokens,
                 cache_write_tokens=cache_write_tokens,
                 error=error,
+                tool_definitions=tool_definitions,
             )
 
         return call_and_trace(original, args, kwargs, on_finish)
@@ -199,6 +201,7 @@ def _patch_stream(
         ctx = original_stream(*args, **kwargs)
         input_repr = _safe_serialize(_prepend_system(kwargs.get("messages"), kwargs.get("system")))
         model = kwargs.get("model")
+        tool_definitions = capture_tool_definitions(kwargs.get("tools"))
 
         def build_and_send(end_t: float, error: Optional[str], final_message: Optional[Any]) -> None:
             output = None
@@ -230,6 +233,7 @@ def _patch_stream(
                 cache_read_tokens=cache_read_tokens,
                 cache_write_tokens=cache_write_tokens,
                 error=error,
+                tool_definitions=tool_definitions,
             )
 
         class _TracedStream:

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx/integrations/google_genai.py

@@ -20,7 +20,7 @@ import time
 from typing import Any, Dict, List, Optional
 
 from agentx.tracing.tracer import Tracer, _safe_serialize
-from agentx.integrations._traced_call import call_and_trace, finish_llm_call
+from agentx.integrations._traced_call import capture_tool_definitions, call_and_trace, finish_llm_call
 
 
 def patch_genai_client(
@@ -90,6 +90,31 @@ def _extract_response_text(response: Any) -> Optional[str]:
     return None
 
 
+
+def _genai_tool_definitions(config: Any) -> Optional[list]:
+    """
+    Flatten a GenAI request's config.tools (each a Tool holding function_declarations) into the
+    flat [{name, description, parameters}, ...] list the engine's unregistered-tool listing
+    reads from trace metadata. Plain dicts and SDK objects both handled.
+    """
+    tools = getattr(config, "tools", None) if config is not None and not isinstance(config, dict) else (config or {}).get("tools") if isinstance(config, dict) else None
+    if not tools:
+        return None
+    flat = []
+    for tool in tools:
+        decls = getattr(tool, "function_declarations", None) if not isinstance(tool, dict) else tool.get("function_declarations")
+        for decl in decls or []:
+            if isinstance(decl, dict):
+                flat.append(decl)
+            else:
+                flat.append({
+                    "name": getattr(decl, "name", None),
+                    "description": getattr(decl, "description", None),
+                    "parameters": getattr(decl, "parameters", None),
+                })
+    return capture_tool_definitions(flat)
+
+
 def _patch_generate_content(
     models: Any,
     tracer: Tracer,
@@ -106,6 +131,7 @@ def _patch_generate_content(
         model = kwargs.get("model") or (args[0] if args else None)
         contents = kwargs.get("contents") or (args[1] if len(args) > 1 else None)
         input_repr = contents if isinstance(contents, str) else _safe_serialize(contents)
+        tool_definitions = _genai_tool_definitions(kwargs.get("config"))
 
         def on_finish(response: Optional[Any], error: Optional[str]) -> None:
             end_t = time.time()
@@ -137,6 +163,7 @@ def _patch_generate_content(
                 output_tokens=output_tokens,
                 cache_read_tokens=cache_read_tokens,
                 error=error,
+                tool_definitions=tool_definitions,
             )
 
         return call_and_trace(original, args, kwargs, on_finish)
@@ -180,6 +207,7 @@ def _patch_sync_generate_content_stream(
     def patched_stream(*args, **kwargs):
         start_t = time.time()
         model, input_repr = _stream_input_and_model(args, kwargs)
+        tool_definitions = _genai_tool_definitions(kwargs.get("config"))
         accumulated_text: List[str] = []
         last_usage_metadata = None
         error: Optional[str] = None
@@ -222,6 +250,7 @@ def _patch_sync_generate_content_stream(
                 output_tokens=output_tokens,
                 cache_read_tokens=cache_read_tokens,
                 error=error,
+                tool_definitions=tool_definitions,
             )
 
     patched_stream._agentx_patched = True
@@ -247,6 +276,7 @@ def _patch_async_generate_content_stream(
         # drives it.
         start_t = time.time()
         model, input_repr = _stream_input_and_model(args, kwargs)
+        tool_definitions = _genai_tool_definitions(kwargs.get("config"))
         inner = await original_stream(*args, **kwargs)
 
         async def traced_agen():
@@ -290,6 +320,7 @@ def _patch_async_generate_content_stream(
                     output_tokens=output_tokens,
                     cache_read_tokens=cache_read_tokens,
                     error=error,
+                    tool_definitions=tool_definitions,
                 )
 
         return traced_agen()

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx/integrations/langchain.py

@@ -26,6 +26,7 @@ from typing import Any, Dict, List, Optional, Union
 from uuid import UUID
 
 from agentx.tracing.tracer import Tracer, _safe_serialize
+from agentx.integrations._traced_call import capture_tool_definitions
 
 try:
     from langchain_core.callbacks.base import BaseCallbackHandler
@@ -375,6 +376,10 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                 # skipped plumbing runs to the nearest emitted ancestor.
                 "node_runs": {},
                 "chain_parents": {},
+                # The request's tools=[...] as seen on the first LLM call's invocation params -
+                # attached to the root trace's metadata so the engine's unregistered-tool
+                # listing can surface the REAL definition (not one inferred from arguments).
+                "tool_definitions": None,
             }
         else:
             top = self._find_top_ancestor(parent_run_id)
@@ -522,6 +527,8 @@ class AgentXCallbackHandler(BaseCallbackHandler):
 
         active_span = self._tracer.current_span
         if active_span is not None:
+            if state.get("tool_definitions") and not (active_span._metadata or {}).get("tools"):
+                active_span._metadata = {**(active_span._metadata or {}), "tools": state["tool_definitions"]}
             # Part of a `with tracer.trace(...)` block (e.g. an orchestrator
             # spanning several chain/agent/retriever calls) - fold this
             # top-level run into it instead of sending an independent trace.
@@ -540,7 +547,15 @@ class AgentXCallbackHandler(BaseCallbackHandler):
             # Standalone usage (no enclosing `with tracer.trace()`): open a real root span for
             # this chain invocation and let _merge_child_run explode its accumulated
             # execution_steps/tool_calls/retrieval_steps into real child-span rows.
-            with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
+            with self._tracer.trace(
+                self._name,
+                metadata=(
+                    {**(self._metadata or {}), "tools": state["tool_definitions"]}
+                    if state.get("tool_definitions")
+                    else self._metadata
+                ),
+                session_id=self._session_id,
+            ) as span:
                 # __enter__ just set _start to "now" - overridden to the chain's real start time,
                 # see llamaindex.py's _send_trace for the identical fix and full rationale.
                 span._start = state["start"]
@@ -577,6 +592,8 @@ class AgentXCallbackHandler(BaseCallbackHandler):
 
         active_span = self._tracer.current_span
         if active_span is not None:
+            if state.get("tool_definitions") and not (active_span._metadata or {}).get("tools"):
+                active_span._metadata = {**(active_span._metadata or {}), "tools": state["tool_definitions"]}
             active_span.set_error(str(error))
             active_span._merge_child_run(
                 tool_calls=state["tool_calls"],
@@ -590,7 +607,15 @@ class AgentXCallbackHandler(BaseCallbackHandler):
             self._emit_span_tree(active_span, state, state["tool_calls"])
         else:
             # See on_chain_end's matching branch - same standalone-usage handling.
-            with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
+            with self._tracer.trace(
+                self._name,
+                metadata=(
+                    {**(self._metadata or {}), "tools": state["tool_definitions"]}
+                    if state.get("tool_definitions")
+                    else self._metadata
+                ),
+                session_id=self._session_id,
+            ) as span:
                 span._start = state["start"]
                 span.set_error(str(error))
                 span._merge_child_run(
@@ -629,6 +654,11 @@ class AgentXCallbackHandler(BaseCallbackHandler):
             or serialized.get("name")
         )
         model = str(model) if model and model not in ("None", "none") else None
+        top_for_tools = self._find_top_ancestor(parent_run_id)
+        if top_for_tools and top_for_tools in self._runs and not self._runs[top_for_tools].get("tool_definitions"):
+            captured = capture_tool_definitions(kwargs.get("invocation_params", {}).get("tools"))
+            if captured:
+                self._runs[top_for_tools]["tool_definitions"] = captured
         self._runs[run_id] = {
             "llm_start": time.time(),
             "model": model,

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx/integrations/litellm.py

@@ -18,7 +18,7 @@ from __future__ import annotations
 from typing import Any, Dict, Optional, Tuple
 
 from agentx.tracing.tracer import Tracer, _safe_serialize
-from agentx.integrations._traced_call import finish_llm_call
+from agentx.integrations._traced_call import capture_tool_definitions, finish_llm_call
 
 try:
     from litellm.integrations.custom_logger import CustomLogger
@@ -98,6 +98,10 @@ class AgentXLiteLLMLogger(CustomLogger):
     def _finish(self, kwargs: Dict[str, Any], response_obj: Any, start_time: Any, end_time: Any, error: Optional[str]) -> None:
         model = kwargs.get("model")
         input_repr = _safe_serialize(kwargs.get("messages"))
+        # LiteLLM surfaces the request's tools under optional_params (and sometimes top-level).
+        tool_definitions = capture_tool_definitions(
+            kwargs.get("tools") or (kwargs.get("optional_params") or {}).get("tools")
+        )
         output = None
         input_tokens = None
         output_tokens = None
@@ -118,6 +122,7 @@ class AgentXLiteLLMLogger(CustomLogger):
             output=output,
             model=model,
             input_tokens=input_tokens,
+            tool_definitions=tool_definitions,
             output_tokens=output_tokens,
             cache_read_tokens=cache_read_tokens,
             error=error,

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx/integrations/openai.py

@@ -28,7 +28,7 @@ import time
 from typing import Any, Dict, Optional, Tuple
 
 from agentx.tracing.tracer import Tracer, _safe_serialize
-from agentx.integrations._traced_call import call_and_trace, finish_llm_call
+from agentx.integrations._traced_call import capture_tool_definitions, call_and_trace, finish_llm_call
 
 
 def _extract_output_text(response: Any) -> Optional[str]:
@@ -128,6 +128,7 @@ def _patch_chat_completions_create(
         input_messages = kwargs.get("messages") or (args[0] if args else None)
         model = kwargs.get("model")
         input_repr = _safe_serialize(input_messages)
+        tool_definitions = capture_tool_definitions(kwargs.get("tools"))
 
         def on_finish(response: Optional[Any], error: Optional[str]) -> None:
             end_t = time.time()
@@ -159,6 +160,7 @@ def _patch_chat_completions_create(
                 output_tokens=output_tokens,
                 cache_read_tokens=cache_read_tokens,
                 error=error,
+                tool_definitions=tool_definitions,
             )
 
         return call_and_trace(original, args, kwargs, on_finish)

agentx_python-0.6.25/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.25"

{agentx_python-0.6.23 → agentx_python-0.6.25}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.23
+Version: 0.6.25
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

agentx_python-0.6.23/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.23"