agentx-python 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

agentx/evaluations/models.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from typing import Any, Dict, List, Literal, Optional, Union
-from pydantic import BaseModel, Field, model_validator
+from pydantic import AliasChoices, BaseModel, Field, model_validator
 
 # ---------------------------------------------------------------------------
 # Observable trace
@@ -418,7 +418,11 @@ class RunResultRow(BaseModel):
     latency_ms: Optional[float] = Field(default=None, alias="latencyMs")
     input_tokens: Optional[int] = Field(default=None, alias="inputTokens")
     output_tokens: Optional[int] = Field(default=None, alias="outputTokens")
-    cosine_similarity: Optional[float] = Field(default=None, alias="cosineSimilarity")
+    # Self-host sends vectorSimilarity, the hosted platform cosineSimilarity - accept both
+    # (row.cosine_similarity was silently None forever on self-host before this).
+    cosine_similarity: Optional[float] = Field(
+        default=None, validation_alias=AliasChoices("cosineSimilarity", "vectorSimilarity")
+    )
     jaccard_similarity: Optional[float] = Field(default=None, alias="jaccardSimilarity")
     bleu_score: Optional[float] = Field(default=None, alias="bleuScore")
     rouge_score: Optional[float] = Field(default=None, alias="rougeScore")
@@ -468,7 +472,11 @@ class ReportStatistics(BaseModel):
     average_rating: float = Field(default=0.0, alias="averageRating")
     min_rating: float = Field(default=0.0, alias="minRating")
     max_rating: float = Field(default=0.0, alias="maxRating")
-    cosine_similarity: Optional[float] = Field(default=None, alias="cosineSimilarity")
+    # Self-host sends vectorSimilarity, the hosted platform cosineSimilarity - accept both
+    # (row.cosine_similarity was silently None forever on self-host before this).
+    cosine_similarity: Optional[float] = Field(
+        default=None, validation_alias=AliasChoices("cosineSimilarity", "vectorSimilarity")
+    )
     jaccard_similarity: Optional[float] = Field(default=None, alias="jaccardSimilarity")
     bleu_score: Optional[float] = Field(default=None, alias="bleuScore")
     rouge_score: Optional[float] = Field(default=None, alias="rougeScore")

agentx/evaluations/runner.py

@@ -119,7 +119,21 @@ class EvaluationRunContext:
     # ------------------------------------------------------------------
 
     def execute(self, adapter: AdapterLike) -> "EvaluationRunContext":
-        """Run all cases locally and submit batches to AgentX."""
+        """Run all cases locally and submit batches to AgentX.
+
+        The whole loop runs inside the eval-run scope (tracing/eval_scope.py): any trace the
+        agent function creates is stamped source="eval-run" + monitor=False automatically, so
+        eval traffic never skews production monitoring and no one has to remember a flag.
+        """
+        from agentx.tracing.eval_scope import enter_eval_run, exit_eval_run
+
+        scope_token = enter_eval_run(self._run.run_id)
+        try:
+            return self._execute_inner(adapter)
+        finally:
+            exit_eval_run(scope_token)
+
+    def _execute_inner(self, adapter: AdapterLike) -> "EvaluationRunContext":
         normalized = _wrap_adapter(adapter)
         cases = _build_cases(self._dataset, self._run, self._evaluation_settings)
         max_batch = self._run.limits.max_batch_size

agentx/tracing/eval_scope.py

@@ -0,0 +1,44 @@
+"""The eval-run scope: how traces created inside an evaluation stop passing as production.
+
+An offline run executes the user's own agent function, and an instrumented agent traces itself -
+which is exactly what makes trajectory matching and retrieval-context extraction work. But those
+traces are not production traffic, and before this scope existed the burden of saying so sat on
+every caller: remember ``monitor=False`` on every ``tracer.trace(...)`` inside an eval, or the
+engine would double-judge each case, raise signals on synthetic questions, and count the run's
+latencies into production KPIs. Nobody remembered - including our own samples.
+
+``EvaluationRunContext.execute()`` enters this scope around the whole run. While it is active,
+every trace the tracer sends is stamped:
+
+  - ``source="eval-run"``  - the engine files it as eval traffic (excluded from monitor KPIs,
+    metrics, sessions and the Live Traces default view; cost keeps it, split out)
+  - ``monitor=False``      - unless the caller explicitly passed ``monitor=True``, which is
+    respected as a deliberate choice
+  - ``metadata.evalRunId`` - so a trace can always be walked back to the run that produced it
+
+A ``contextvars.ContextVar`` rather than tracer state: it nests correctly, cannot leak across
+concurrent runs in async code, and costs nothing when no run is active. The one known limit is
+threads the agent function spawns itself - a context var does not cross a bare ``Thread()`` -
+which matches the tracer's existing documented posture for user-managed threads.
+"""
+
+from contextvars import ContextVar
+from typing import Optional
+
+EVAL_RUN_SOURCE = "eval-run"
+
+_current_eval_run_id: ContextVar[Optional[str]] = ContextVar("agentx_eval_run_id", default=None)
+
+
+def enter_eval_run(run_id: str):
+    """Mark the current context as inside an eval run. Returns the token for ``exit_eval_run``."""
+    return _current_eval_run_id.set(run_id)
+
+
+def exit_eval_run(token) -> None:
+    _current_eval_run_id.reset(token)
+
+
+def current_eval_run_id() -> Optional[str]:
+    """The run id when inside ``execute()``, else None."""
+    return _current_eval_run_id.get()

agentx/tracing/tracer.py

@@ -13,6 +13,7 @@ from uuid import uuid4
 from agentx.exceptions import CIGateFailure
 from agentx.tracing.ingest_client import IngestClient
 from agentx.tracing.ci_types import CIRun, CIRunResult, CIRunStatus, CIQuestionScore
+from agentx.tracing.eval_scope import EVAL_RUN_SOURCE, current_eval_run_id
 
 F = TypeVar("F", bound=Callable[..., Any])
 
@@ -167,9 +168,21 @@ class _TraceSpan:
             # flush() uses; child-only spans keep their async fire-and-forget behavior.
             self._tracer.flush(timeout=5.0)
 
+        # Inside an eval run (evaluations' execute()), every trace states what it is: eval
+        # traffic. monitor=False unless the caller explicitly said True; the run id rides in
+        # metadata so the trace can be walked back to its run. See tracing/eval_scope.py.
+        eval_run_id = current_eval_run_id()
+        monitor = self._monitor
+        metadata = self._metadata
+        source = None
+        if eval_run_id is not None:
+            source = EVAL_RUN_SOURCE
+            if monitor is not True:
+                monitor = False
+            metadata = {**(metadata or {}), "evalRunId": eval_run_id}
         self._trace_id = self._tracer._send(
             sync=self._sync,
-            monitor=self._monitor,
+            monitor=monitor,
             pattern_ids=self._pattern_ids,
             name=self.name,
             agent_id=self._agent_id,
@@ -177,7 +190,7 @@ class _TraceSpan:
             output=_safe_serialize(self.output) if self.output is not None else None,
             latency_ms=latency_ms,
             error=self._error,
-            metadata=self._metadata,
+            metadata=metadata,
             framework=self._framework or self._captured_framework,
             model=self._model or self._captured_model,
             tool_calls=self.tool_calls or None,
@@ -189,6 +202,7 @@ class _TraceSpan:
             span_id=self._span_id,
             parent_span_id=self._parent_span_id,
             span_kind=self._span_kind,
+            source=source,
             started_at_unix_nano=str(int(self._start * 1_000_000_000)) if self._start else None,
         )
         return False  # never suppress exceptions
@@ -332,6 +346,8 @@ class _TraceSpan:
             wire["span_kind"] = span_kind
         if child._session_id:
             wire["session_id"] = child._session_id
+        if current_eval_run_id() is not None:
+            wire["source"] = EVAL_RUN_SOURCE
         wire["span_id"] = child._span_id
         if child._parent_span_id:
             wire["parent_span_id"] = child._parent_span_id
@@ -1148,6 +1164,8 @@ class Tracer:
             wire["started_at_unix_nano"] = payload["started_at_unix_nano"]
         if "agent_id" in payload:
             wire["agent_id"] = payload["agent_id"]
+        if "source" in payload:
+            wire["source"] = payload["source"]
         if "span_kind" in payload:
             wire["span_kind"] = payload["span_kind"]
 

agentx/version.py

@@ -1 +1 @@
-VERSION = "0.8.2"
+VERSION = "0.8.4"

{agentx_python-0.8.2.dist-info → agentx_python-0.8.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.8.2
+Version: 0.8.4
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.8.2.dist-info → agentx_python-0.8.4.dist-info}/RECORD

@@ -10,17 +10,17 @@ agentx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 agentx/testing.py,sha256=VKry0lgqaeMyxy9aPuSV73LTUhRv0WcInVVWOdkz4e0,7194
 agentx/traces.py,sha256=sz9gxutlDKNgf0fdCkyzKsGwK5eMAJuIEl3NIwGrsuo,2287
 agentx/util.py,sha256=lt2Kpg4Fj8whrSuAhLEahcPEIYRVpK9RCC5y9ShdTJs,703
-agentx/version.py,sha256=1VXNrsOvWIerd1gj2Z41YiUd-fiUy2DWAvpw33fR7ig,18
+agentx/version.py,sha256=IExNVK5GJVnQo4b-FGsI7pTg9u4gzQTgCzUF5lFpZ_I,18
 agentx/evaluations/__init__.py,sha256=Erv7RGFlRxGTG4rVb2uHhCqLLEX_iAWKqkZKzK6CumE,262
 agentx/evaluations/_term.py,sha256=WFpiNzdgDBeJJ-Gg-6X7TwwxllobuE4OqFUTuDQvS3Y,2529
 agentx/evaluations/client.py,sha256=1MsduXHOSkwS40RvbUpNgGMU4pZvFtChqlU82YqKdOw,30028
 agentx/evaluations/datasets.py,sha256=7OYj5pXpJenViAhPe7Sr_7Sf9KWd7b1OhV-dPL2tso8,14283
 agentx/evaluations/evaluation_settings.py,sha256=geRydHqKpxwkxBVfMIOqkB6NlStyOZrYwZnMHluDwvA,6276
-agentx/evaluations/models.py,sha256=Q2oV4n5DPlWGY68a1D50AYJuAzBkjCEe1RoyFs4eLhY,31221
+agentx/evaluations/models.py,sha256=b8cp0arikLW6tZqzmRs-4DVqgFs9FSR5DU9LleO82VY,31699
 agentx/evaluations/prompts.py,sha256=8xyvMpAl3mD5xuSZozuaohdnCBWe5Q02RldwiPLUO20,3493
 agentx/evaluations/reporting.py,sha256=GtNnL-1eNEQrSqd0yrHi5mGqC9_u0kUAN8Mbjdhw6yY,6389
 agentx/evaluations/results.py,sha256=w7TRUik4eXIqrznXvlF_L9hnObnHvgdRKvihfN1XeAU,4692
-agentx/evaluations/runner.py,sha256=jPUPpuziRLUc8m_wbWwUvUrdwfSZ0cMRRnTdUSY-qsY,27332
+agentx/evaluations/runner.py,sha256=6k-fUYAwf_sbxSWlirQzI7D60ODjSfvGUKeLXnSKD_Q,27947
 agentx/evaluations/tool_schemas.py,sha256=ZyrnSOnx9xlSAj4swfDey9ZLS2ly-g21DByTwRAGwNA,2323
 agentx/evaluations/tracing.py,sha256=MSJD9bzfInMi7E70mc8mWUc0fx5DLkOgUzRru2mNQoc,1825
 agentx/evaluations/adapters/__init__.py,sha256=fK8Hx75usbiY03XUSmnLnSrwBXip2CQs24YZcdEgj6w,287
@@ -58,11 +58,12 @@ agentx/resources/conversation.py,sha256=94pOdM6Pmj67RhikZVBNGkT3V-WKK_ZAD7s27p5-
 agentx/resources/workforce.py,sha256=lfGVkoV9lcOp-lZScjTvwRarJMkhzfMfLm4syJDujGc,4168
 agentx/tracing/__init__.py,sha256=l2mIRILE-wWrCD0fekgrIOQUEvtDObzdrkgpQUtXroU,408
 agentx/tracing/ci_types.py,sha256=zFVcZGvc1qbVDdOlEPQ1i5R6terqJHxCzFF4ZjouxxI,1423
+agentx/tracing/eval_scope.py,sha256=ElMbPxpqpQBVuaUnHNlw9RIQu71oyOpd0yR55bDH8eI,2144
 agentx/tracing/ingest_client.py,sha256=5X6RyxtMq1XYgOzsMd3b2f2qpmYpsMnhKm7NB3oup00,17664
-agentx/tracing/tracer.py,sha256=iZymguMO3TOoS1YDbS6RfEdt7CWWavA3HO4iiSnnVbw,52893
-agentx_python-0.8.2.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
-agentx_python-0.8.2.dist-info/METADATA,sha256=eoN7qIXyCrvWO_HYlipOJ-bAGFGxOSh_y8mG46x81hw,20992
-agentx_python-0.8.2.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-agentx_python-0.8.2.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
-agentx_python-0.8.2.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
-agentx_python-0.8.2.dist-info/RECORD,,
+agentx/tracing/tracer.py,sha256=qQojU-T4O8IJKyf9vue0Em66qPBTZKl3FMyr3KJKPnI,53770
+agentx_python-0.8.4.dist-info/licenses/LICENSE,sha256=gZVsM-nLsE8vlaY6NXXsVoo6IlCClxkToAWmhgT3y_s,10762
+agentx_python-0.8.4.dist-info/METADATA,sha256=g4P1EOltG3-fpNZuo9lmVGA3UhcFuxnYQxj1l-57X44,20992
+agentx_python-0.8.4.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+agentx_python-0.8.4.dist-info/entry_points.txt,sha256=rQqF1JTY3T1yfviBU1r8PU-5mBdfOk6P4bZ1L4BskIM,172
+agentx_python-0.8.4.dist-info/top_level.txt,sha256=s-q-HB9Gb_QdrZNacSeQyF_c25gQooMy7DlxzgLOHPk,7
+agentx_python-0.8.4.dist-info/RECORD,,