agentx-python 0.6.24__tar.gz → 0.6.25__tar.gz

{agentx_python-0.6.24 → agentx_python-0.6.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.24
+Version: 0.6.25
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.24 → agentx_python-0.6.25}/agentx/evaluations/datasets.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import csv
 import logging
 from pathlib import Path
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union
 
 from agentx.evaluations.models import Dataset
 
@@ -88,6 +88,7 @@ class DatasetBuilder:
         smoke_test_guidance: Optional[str] = None,
         expected_tools: Optional[List[str]] = None,
         trajectory_match_mode: str = "strict",
+        expected_retrieval_context: Optional[Union[str, List[str]]] = None,
     ) -> "DatasetBuilder":
         """Add a case. `judge_guideline` is optional grading guidance specific to this question.
 
@@ -99,6 +100,13 @@ class DatasetBuilder:
         any order), "superset" (all expected present, extras allowed), or "subset" (no unexpected
         calls, missing allowed).
 
+        `expected_retrieval_context` (string or list of chunk strings) declares what a correct
+        retriever should have fetched for this case. When the run's result carries actual
+        retrieved context (a `retrieval_context` return value, or a linked trace with retrieval
+        spans), the engine compares the two with token-level Jaccard similarity and reports a
+        deterministic "Context match (jaccard)" scorer row (0-1) - a cheap retriever regression
+        check with no LLM judge call.
+
         `smoke_test_count`, when set (1-10), asks this question that many extra ways each
         evaluation run, LLM-paraphrased server-side, to catch agents that are brittle to phrasing
         rather than genuinely wrong. `smoke_test_guidance` optionally steers what kind of variants
@@ -123,6 +131,8 @@ class DatasetBuilder:
                 main["smokeTest"]["guidance"] = smoke_test_guidance
         if expected_tools:
             main["expectedTrajectory"] = {"tools": expected_tools, "mode": trajectory_match_mode}
+        if expected_retrieval_context:
+            main["expectedRetrievalContext"] = expected_retrieval_context
         self._payload["questions"].append(
             {
                 "main_question": main,

{agentx_python-0.6.24 → agentx_python-0.6.25}/agentx/evaluations/models.py

@@ -346,6 +346,11 @@ class EvaluationResult(BaseModel):
     # the dashboard's "Message Trace Details -> Execution Timeline" render the full execution
     # trace for this case, not just the lightweight observable_trace events above.
     trace_id: Optional[str] = Field(default=None, alias="traceId")
+    # What the agent actually retrieved for THIS case (a string, or list of chunk strings) - the
+    # {context} the RAG metric pack's judges grade faithfulness/relevancy against. Overrides the
+    # question's static retrievalContext; when absent, the engine falls back to the linked
+    # trace's recorded retrieval spans (trace_id above), then the case's pinned context.
+    retrieval_context: Optional[Any] = Field(default=None, alias="retrievalContext")
     # Smoke test: set by execute() from the originating EvaluationCase, not something you need to
     # set yourself when returning a plain str/dict from your callable.
     is_smoke_test_variant: Optional[bool] = Field(default=None, alias="isSmokeTestVariant")

{agentx_python-0.6.24 → agentx_python-0.6.25}/agentx/evaluations/results.py

@@ -37,6 +37,7 @@ def normalize_result(
     output: Optional[dict] = None
     trace = None
     trace_id: Optional[str] = None
+    retrieval_context = None
     metadata: Optional[dict] = None
     error: Optional[ResultError] = None
     input_tokens: Optional[int] = None
@@ -55,6 +56,7 @@ def normalize_result(
         trace = build_trace(raw.get("trace") or raw.get("observable_trace"))
         trace_id_raw = raw.get("trace_id") or raw.get("traceId")
         trace_id = str(trace_id_raw) if trace_id_raw else None
+        retrieval_context = raw.get("retrieval_context") or raw.get("retrievalContext")
         meta_raw = raw.get("metadata")
         if isinstance(meta_raw, dict):
             metadata = redact_dict(meta_raw)
@@ -108,6 +110,7 @@ def normalize_result(
         ),
         metadata=metadata,
         traceId=trace_id,
+        retrievalContext=retrieval_context,
         isSmokeTestVariant=case.is_smoke_test_variant,
         smokeTestVariantText=case.smoke_test_variant_text,
     )

agentx_python-0.6.25/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.25"

{agentx_python-0.6.24 → agentx_python-0.6.25}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.24
+Version: 0.6.25
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

agentx_python-0.6.24/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.24"