agentx-python 0.6.24__tar.gz → 0.6.26__tar.gz

{agentx_python-0.6.24 → agentx_python-0.6.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.24
+Version: 0.6.26
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.24 → agentx_python-0.6.26}/agentx/evaluations/datasets.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import csv
 import logging
 from pathlib import Path
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union
 
 from agentx.evaluations.models import Dataset
 
@@ -88,6 +88,7 @@ class DatasetBuilder:
         smoke_test_guidance: Optional[str] = None,
         expected_tools: Optional[List[str]] = None,
         trajectory_match_mode: str = "strict",
+        expected_retrieval_context: Optional[Union[str, List[str]]] = None,
     ) -> "DatasetBuilder":
         """Add a case. `judge_guideline` is optional grading guidance specific to this question.
 
@@ -99,6 +100,13 @@ class DatasetBuilder:
         any order), "superset" (all expected present, extras allowed), or "subset" (no unexpected
         calls, missing allowed).
 
+        `expected_retrieval_context` (string or list of chunk strings) declares what a correct
+        retriever should have fetched for this case. When the run's result carries actual
+        retrieved context (a `retrieval_context` return value, or a linked trace with retrieval
+        spans), the engine compares the two with token-level Jaccard similarity and reports a
+        deterministic "Context match (jaccard)" scorer row (0-1) - a cheap retriever regression
+        check with no LLM judge call.
+
         `smoke_test_count`, when set (1-10), asks this question that many extra ways each
         evaluation run, LLM-paraphrased server-side, to catch agents that are brittle to phrasing
         rather than genuinely wrong. `smoke_test_guidance` optionally steers what kind of variants
@@ -123,6 +131,8 @@ class DatasetBuilder:
                 main["smokeTest"]["guidance"] = smoke_test_guidance
         if expected_tools:
             main["expectedTrajectory"] = {"tools": expected_tools, "mode": trajectory_match_mode}
+        if expected_retrieval_context:
+            main["expectedRetrievalContext"] = expected_retrieval_context
         self._payload["questions"].append(
             {
                 "main_question": main,

{agentx_python-0.6.24 → agentx_python-0.6.26}/agentx/evaluations/models.py

@@ -346,6 +346,11 @@ class EvaluationResult(BaseModel):
     # the dashboard's "Message Trace Details -> Execution Timeline" render the full execution
     # trace for this case, not just the lightweight observable_trace events above.
     trace_id: Optional[str] = Field(default=None, alias="traceId")
+    # What the agent actually retrieved for THIS case (a string, or list of chunk strings) - the
+    # {context} the RAG metric pack's judges grade faithfulness/relevancy against. Overrides the
+    # question's static retrievalContext; when absent, the engine falls back to the linked
+    # trace's recorded retrieval spans (trace_id above), then the case's pinned context.
+    retrieval_context: Optional[Any] = Field(default=None, alias="retrievalContext")
     # Smoke test: set by execute() from the originating EvaluationCase, not something you need to
     # set yourself when returning a plain str/dict from your callable.
     is_smoke_test_variant: Optional[bool] = Field(default=None, alias="isSmokeTestVariant")

{agentx_python-0.6.24 → agentx_python-0.6.26}/agentx/evaluations/results.py

@@ -37,6 +37,7 @@ def normalize_result(
     output: Optional[dict] = None
     trace = None
     trace_id: Optional[str] = None
+    retrieval_context = None
     metadata: Optional[dict] = None
     error: Optional[ResultError] = None
     input_tokens: Optional[int] = None
@@ -55,6 +56,7 @@ def normalize_result(
         trace = build_trace(raw.get("trace") or raw.get("observable_trace"))
         trace_id_raw = raw.get("trace_id") or raw.get("traceId")
         trace_id = str(trace_id_raw) if trace_id_raw else None
+        retrieval_context = raw.get("retrieval_context") or raw.get("retrievalContext")
         meta_raw = raw.get("metadata")
         if isinstance(meta_raw, dict):
             metadata = redact_dict(meta_raw)
@@ -108,6 +110,7 @@ def normalize_result(
         ),
         metadata=metadata,
         traceId=trace_id,
+        retrievalContext=retrieval_context,
         isSmokeTestVariant=case.is_smoke_test_variant,
         smokeTestVariantText=case.smoke_test_variant_text,
     )

{agentx_python-0.6.24 → agentx_python-0.6.26}/agentx/integrations/langchain.py

@@ -489,6 +489,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
                 duration_ms=step.get("duration_ms"),
                 input=step.get("query"),
                 output=step.get("output"),
+                metadata={"kind": "retrieval"},
             )
 
     def on_chain_end(

{agentx_python-0.6.24 → agentx_python-0.6.26}/agentx/tracing/tracer.py

@@ -252,6 +252,7 @@ class _TraceSpan:
         cache_write_tokens: Optional[int] = None,
         error: Optional[str] = None,
         tool_calls: Optional[List[Dict[str, Any]]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
     ) -> "_TraceSpan":
         """
         Send one real child-span row parented to this span, with explicit timing (the caller's
@@ -306,6 +307,8 @@ class _TraceSpan:
             wire["model"] = model
         if child.tool_calls:
             wire["tool_calls"] = child.tool_calls
+        if metadata:
+            wire["metadata"] = _safe_serialize(metadata)
         if child._session_id:
             wire["session_id"] = child._session_id
         wire["span_id"] = child._span_id
@@ -740,8 +743,18 @@ class Tracer:
         active_span = self.current_span
         if active_span is None:
             return
+        # The kind marker is what tells the engine (retrieval-context extraction for RAG
+        # judges) and the dashboard timeline that this span is a retrieval regardless of its
+        # name - the name-based "Retrieval N" heuristic remains only as a fallback for older
+        # traces, so custom names like "kb_search" work everywhere.
         active_span.child_span(
-            name, start_time=start_time, end_time=end_time, duration_ms=duration_ms, input=query, output=output
+            name,
+            start_time=start_time,
+            end_time=end_time,
+            duration_ms=duration_ms,
+            input=query,
+            output=output,
+            metadata={"kind": "retrieval"},
         )
 
     @contextmanager

agentx_python-0.6.26/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.26"

{agentx_python-0.6.24 → agentx_python-0.6.26}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.24
+Version: 0.6.26
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

agentx_python-0.6.24/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.24"