agentx-python 0.8.2__tar.gz → 0.8.4__tar.gz

{agentx_python-0.8.2/agentx_python.egg-info → agentx_python-0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.2
+Version: 0.8.4
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.8.2 → agentx_python-0.8.4}/agentx/evaluations/models.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from typing import Any, Dict, List, Literal, Optional, Union
-from pydantic import BaseModel, Field, model_validator
+from pydantic import AliasChoices, BaseModel, Field, model_validator
 
 # ---------------------------------------------------------------------------
 # Observable trace
@@ -418,7 +418,11 @@ class RunResultRow(BaseModel):
     latency_ms: Optional[float] = Field(default=None, alias="latencyMs")
     input_tokens: Optional[int] = Field(default=None, alias="inputTokens")
     output_tokens: Optional[int] = Field(default=None, alias="outputTokens")
-    cosine_similarity: Optional[float] = Field(default=None, alias="cosineSimilarity")
+    # Self-host sends vectorSimilarity, the hosted platform cosineSimilarity - accept both
+    # (row.cosine_similarity was silently None forever on self-host before this).
+    cosine_similarity: Optional[float] = Field(
+        default=None, validation_alias=AliasChoices("cosineSimilarity", "vectorSimilarity")
+    )
     jaccard_similarity: Optional[float] = Field(default=None, alias="jaccardSimilarity")
     bleu_score: Optional[float] = Field(default=None, alias="bleuScore")
     rouge_score: Optional[float] = Field(default=None, alias="rougeScore")
@@ -468,7 +472,11 @@ class ReportStatistics(BaseModel):
     average_rating: float = Field(default=0.0, alias="averageRating")
     min_rating: float = Field(default=0.0, alias="minRating")
     max_rating: float = Field(default=0.0, alias="maxRating")
-    cosine_similarity: Optional[float] = Field(default=None, alias="cosineSimilarity")
+    # Self-host sends vectorSimilarity, the hosted platform cosineSimilarity - accept both
+    # (row.cosine_similarity was silently None forever on self-host before this).
+    cosine_similarity: Optional[float] = Field(
+        default=None, validation_alias=AliasChoices("cosineSimilarity", "vectorSimilarity")
+    )
     jaccard_similarity: Optional[float] = Field(default=None, alias="jaccardSimilarity")
     bleu_score: Optional[float] = Field(default=None, alias="bleuScore")
     rouge_score: Optional[float] = Field(default=None, alias="rougeScore")

{agentx_python-0.8.2 → agentx_python-0.8.4}/agentx/evaluations/runner.py

@@ -119,7 +119,21 @@ class EvaluationRunContext:
     # ------------------------------------------------------------------
 
     def execute(self, adapter: AdapterLike) -> "EvaluationRunContext":
-        """Run all cases locally and submit batches to AgentX."""
+        """Run all cases locally and submit batches to AgentX.
+
+        The whole loop runs inside the eval-run scope (tracing/eval_scope.py): any trace the
+        agent function creates is stamped source="eval-run" + monitor=False automatically, so
+        eval traffic never skews production monitoring and no one has to remember a flag.
+        """
+        from agentx.tracing.eval_scope import enter_eval_run, exit_eval_run
+
+        scope_token = enter_eval_run(self._run.run_id)
+        try:
+            return self._execute_inner(adapter)
+        finally:
+            exit_eval_run(scope_token)
+
+    def _execute_inner(self, adapter: AdapterLike) -> "EvaluationRunContext":
         normalized = _wrap_adapter(adapter)
         cases = _build_cases(self._dataset, self._run, self._evaluation_settings)
         max_batch = self._run.limits.max_batch_size

agentx_python-0.8.4/agentx/tracing/eval_scope.py

@@ -0,0 +1,44 @@
+"""The eval-run scope: how traces created inside an evaluation stop passing as production.
+
+An offline run executes the user's own agent function, and an instrumented agent traces itself -
+which is exactly what makes trajectory matching and retrieval-context extraction work. But those
+traces are not production traffic, and before this scope existed the burden of saying so sat on
+every caller: remember ``monitor=False`` on every ``tracer.trace(...)`` inside an eval, or the
+engine would double-judge each case, raise signals on synthetic questions, and count the run's
+latencies into production KPIs. Nobody remembered - including our own samples.
+
+``EvaluationRunContext.execute()`` enters this scope around the whole run. While it is active,
+every trace the tracer sends is stamped:
+
+  - ``source="eval-run"``  - the engine files it as eval traffic (excluded from monitor KPIs,
+    metrics, sessions and the Live Traces default view; cost keeps it, split out)
+  - ``monitor=False``      - unless the caller explicitly passed ``monitor=True``, which is
+    respected as a deliberate choice
+  - ``metadata.evalRunId`` - so a trace can always be walked back to the run that produced it
+
+A ``contextvars.ContextVar`` rather than tracer state: it nests correctly, cannot leak across
+concurrent runs in async code, and costs nothing when no run is active. The one known limit is
+threads the agent function spawns itself - a context var does not cross a bare ``Thread()`` -
+which matches the tracer's existing documented posture for user-managed threads.
+"""
+
+from contextvars import ContextVar
+from typing import Optional
+
+EVAL_RUN_SOURCE = "eval-run"
+
+_current_eval_run_id: ContextVar[Optional[str]] = ContextVar("agentx_eval_run_id", default=None)
+
+
+def enter_eval_run(run_id: str):
+    """Mark the current context as inside an eval run. Returns the token for ``exit_eval_run``."""
+    return _current_eval_run_id.set(run_id)
+
+
+def exit_eval_run(token) -> None:
+    _current_eval_run_id.reset(token)
+
+
+def current_eval_run_id() -> Optional[str]:
+    """The run id when inside ``execute()``, else None."""
+    return _current_eval_run_id.get()

{agentx_python-0.8.2 → agentx_python-0.8.4}/agentx/tracing/tracer.py

@@ -13,6 +13,7 @@ from uuid import uuid4
 from agentx.exceptions import CIGateFailure
 from agentx.tracing.ingest_client import IngestClient
 from agentx.tracing.ci_types import CIRun, CIRunResult, CIRunStatus, CIQuestionScore
+from agentx.tracing.eval_scope import EVAL_RUN_SOURCE, current_eval_run_id
 
 F = TypeVar("F", bound=Callable[..., Any])
 
@@ -167,9 +168,21 @@ class _TraceSpan:
             # flush() uses; child-only spans keep their async fire-and-forget behavior.
             self._tracer.flush(timeout=5.0)
 
+        # Inside an eval run (evaluations' execute()), every trace states what it is: eval
+        # traffic. monitor=False unless the caller explicitly said True; the run id rides in
+        # metadata so the trace can be walked back to its run. See tracing/eval_scope.py.
+        eval_run_id = current_eval_run_id()
+        monitor = self._monitor
+        metadata = self._metadata
+        source = None
+        if eval_run_id is not None:
+            source = EVAL_RUN_SOURCE
+            if monitor is not True:
+                monitor = False
+            metadata = {**(metadata or {}), "evalRunId": eval_run_id}
         self._trace_id = self._tracer._send(
             sync=self._sync,
-            monitor=self._monitor,
+            monitor=monitor,
             pattern_ids=self._pattern_ids,
             name=self.name,
             agent_id=self._agent_id,
@@ -177,7 +190,7 @@ class _TraceSpan:
             output=_safe_serialize(self.output) if self.output is not None else None,
             latency_ms=latency_ms,
             error=self._error,
-            metadata=self._metadata,
+            metadata=metadata,
             framework=self._framework or self._captured_framework,
             model=self._model or self._captured_model,
             tool_calls=self.tool_calls or None,
@@ -189,6 +202,7 @@ class _TraceSpan:
             span_id=self._span_id,
             parent_span_id=self._parent_span_id,
             span_kind=self._span_kind,
+            source=source,
             started_at_unix_nano=str(int(self._start * 1_000_000_000)) if self._start else None,
         )
         return False  # never suppress exceptions
@@ -332,6 +346,8 @@ class _TraceSpan:
             wire["span_kind"] = span_kind
         if child._session_id:
             wire["session_id"] = child._session_id
+        if current_eval_run_id() is not None:
+            wire["source"] = EVAL_RUN_SOURCE
         wire["span_id"] = child._span_id
         if child._parent_span_id:
             wire["parent_span_id"] = child._parent_span_id
@@ -1148,6 +1164,8 @@ class Tracer:
             wire["started_at_unix_nano"] = payload["started_at_unix_nano"]
         if "agent_id" in payload:
             wire["agent_id"] = payload["agent_id"]
+        if "source" in payload:
+            wire["source"] = payload["source"]
         if "span_kind" in payload:
             wire["span_kind"] = payload["span_kind"]
 

agentx_python-0.8.4/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.8.4"

{agentx_python-0.8.2 → agentx_python-0.8.4/agentx_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.2
+Version: 0.8.4
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.8.2 → agentx_python-0.8.4}/agentx_python.egg-info/SOURCES.txt

@@ -61,6 +61,7 @@ agentx/resources/conversation.py
 agentx/resources/workforce.py
 agentx/tracing/__init__.py
 agentx/tracing/ci_types.py
+agentx/tracing/eval_scope.py
 agentx/tracing/ingest_client.py
 agentx/tracing/tracer.py
 agentx_python.egg-info/PKG-INFO
@@ -72,6 +73,7 @@ agentx_python.egg-info/requires.txt
 agentx_python.egg-info/top_level.txt
 tests/test_deep_dive_fixes.py
 tests/test_docs_match_sdk.py
+tests/test_eval_scope.py
 tests/test_integration.py
 tests/test_integrations.py
 tests/test_judge_scorers.py

agentx_python-0.8.4/tests/test_eval_scope.py

@@ -0,0 +1,116 @@
+"""The eval-run scope: traces created inside execute() stop passing as production.
+
+Before this, the burden sat on every caller: remember monitor=False on every trace inside an
+eval or the engine double-judges, raises signals on synthetic questions, and counts eval
+latencies into production KPIs. These pin that the stamping is automatic, respects an explicit
+monitor=True, and vanishes completely outside the scope.
+"""
+
+from agentx.tracing.eval_scope import current_eval_run_id, enter_eval_run, exit_eval_run
+from agentx.tracing.tracer import Tracer
+
+
+class _CaptureTracer(Tracer):
+    """A tracer whose network is a list."""
+
+    def __init__(self):  # noqa: D401 - bypass real client setup
+        self.sent = []
+        self._active_spans = []
+        self._pending_tool_calls = []
+        self._pending_retrievals = []
+
+    def _send(self, sync=False, **kwargs):
+        self.sent.append({k: v for k, v in kwargs.items() if v is not None})
+        return "trace-1"
+
+    def _dispatch(self, wire, *, sync=False):
+        self.sent.append(wire)
+        return "trace-child"
+
+    # The bits of Tracer the span touches.
+    def _push_active_span(self, span):
+        self._active_spans.append(span)
+
+    def _pop_active_span(self, span):
+        if span in self._active_spans:
+            self._active_spans.remove(span)
+
+    @property
+    def current_span(self):
+        return self._active_spans[-1] if self._active_spans else None
+
+    def flush(self, timeout=5.0):
+        return True
+
+
+def test_outside_the_scope_nothing_changes():
+    tracer = _CaptureTracer()
+    with tracer.trace("prod-agent", input={"q": "hi"}) as span:
+        span.output = "hello"
+    wire = tracer.sent[-1]
+    assert "source" not in wire
+    assert "monitor" not in wire  # None is filtered out, same as before
+    assert current_eval_run_id() is None
+
+
+def test_inside_the_scope_traces_are_stamped():
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    try:
+        with tracer.trace("agent-under-eval", input={"q": "case 1"}) as span:
+            span.output = "answer"
+    finally:
+        exit_eval_run(token)
+    wire = tracer.sent[-1]
+    assert wire["source"] == "eval-run"
+    assert wire["monitor"] is False
+    assert wire["metadata"]["evalRunId"] == "run-42"
+
+
+def test_explicit_monitor_true_is_respected():
+    # Someone deliberately pointing checks at eval traffic is a choice, not a mistake.
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    try:
+        with tracer.trace("agent-under-eval", monitor=True) as span:
+            span.output = "x"
+    finally:
+        exit_eval_run(token)
+    wire = tracer.sent[-1]
+    assert wire["source"] == "eval-run"
+    assert wire["monitor"] is True
+
+
+def test_child_spans_are_eval_traffic_too():
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    try:
+        with tracer.trace("agent-under-eval") as span:
+            span.child_span("kb_search", output=["chunk"])
+            span.output = "x"
+    finally:
+        exit_eval_run(token)
+    child = next(w for w in tracer.sent if w.get("name") == "kb_search")
+    assert child["source"] == "eval-run"
+
+
+def test_the_scope_does_not_leak():
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    exit_eval_run(token)
+    with tracer.trace("prod-again") as span:
+        span.output = "y"
+    assert "source" not in tracer.sent[-1]
+
+
+def test_caller_metadata_survives_the_stamp():
+    tracer = _CaptureTracer()
+    token = enter_eval_run("run-42")
+    try:
+        with tracer.trace("agent", metadata={"promptName": "support-v3"}) as span:
+            span.output = "x"
+    finally:
+        exit_eval_run(token)
+    md = tracer.sent[-1]["metadata"]
+    assert md["promptName"] == "support-v3"
+    assert md["evalRunId"] == "run-42"

agentx_python-0.8.2/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.8.2"