agentevals-cli 0.9.7__tar.gz → 0.9.8__tar.gz

{agentevals_cli-0.9.7 → agentevals_cli-0.9.8}/.github/workflows/ci.yml

@@ -29,6 +29,9 @@ jobs:
         with:
           enable-cache: true
 
+      - name: Verify uv.lock matches pyproject.toml
+        run: uv lock --check
+
       - name: Install dependencies
         run: |
           uv venv
@@ -62,7 +65,7 @@ jobs:
         run: |
           uv venv --python ${{ matrix.python-version }}
           uv pip install setuptools
-          uv sync --dev --python ${{ matrix.python-version }}
+          uv sync --dev --all-extras --frozen --python ${{ matrix.python-version }}
 
       - name: Run tests
         run: uv run pytest -m "not integration and not e2e" --tb=short -q

{agentevals_cli-0.9.7 → agentevals_cli-0.9.8}/Dockerfile

@@ -32,6 +32,12 @@ ARG VERSION
 ENV SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}
 
 RUN uv sync --frozen --no-dev --extra live --extra postgres --extra kubernetes \
+    # The runtime only uses the uv-managed venv; drop the base image's bundled
+    # pip so its vendored packages (msgpack, pkg_resources) don't ship unused.
+    && rm -rf /usr/local/lib/python*/site-packages/pip \
+              /usr/local/lib/python*/site-packages/pip-*.dist-info \
+              /usr/local/bin/pip* \
+    && ! /usr/local/bin/python -c "import pip" 2>/dev/null \
     && groupadd --gid 1000 app \
     && useradd --uid 1000 --gid app --home-dir /app --no-log-init app \
     && chown -R app:app /app

{agentevals_cli-0.9.7 → agentevals_cli-0.9.8}/PKG-INFO

@@ -1,24 +1,34 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: agentevals-cli
-Version: 0.9.7
+Version: 0.9.8
 Summary: Standalone framework to evaluate agent correctness based on portable OpenTelemetry traces
 License-File: LICENSE
 Requires-Python: >=3.11
+Requires-Dist: aiohttp>=3.14.3
+Requires-Dist: authlib>=1.7.2
 Requires-Dist: click>=8.0
 Requires-Dist: fastapi>=0.115.0
 Requires-Dist: google-adk[eval]<2.2,>=2.1.0
 Requires-Dist: httpx>=0.27.0
+Requires-Dist: idna>=3.18
+Requires-Dist: litellm>=1.85.7
+Requires-Dist: nltk>=3.10.2
 Requires-Dist: opentelemetry-proto>=1.36.0
-Requires-Dist: python-dotenv>=1.0.0
-Requires-Dist: python-multipart>=0.0.12
+Requires-Dist: pyasn1>=0.6.4
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: python-multipart>=0.0.32
 Requires-Dist: pyyaml>=6.0
+Requires-Dist: requests>=2.34.2
 Requires-Dist: tabulate>=0.9.0
+Requires-Dist: urllib3>=2.7.0
 Requires-Dist: uvicorn[standard]>=0.32.0
 Provides-Extra: kubernetes
 Requires-Dist: kubernetes>=36.0.0; extra == 'kubernetes'
 Provides-Extra: live
 Requires-Dist: httpx>=0.27.0; extra == 'live'
-Requires-Dist: mcp>=1.26.0; extra == 'live'
+Requires-Dist: mcp<2,>=1.28.1; extra == 'live'
+Requires-Dist: pydantic-settings>=2.15.0; extra == 'live'
+Requires-Dist: pyjwt>=2.13.0; extra == 'live'
 Provides-Extra: openai
 Requires-Dist: openai>=2.0; extra == 'openai'
 Provides-Extra: postgres

{agentevals_cli-0.9.7 → agentevals_cli-0.9.8}/pyproject.toml

@@ -14,17 +14,30 @@ dependencies = [
     "tabulate>=0.9.0",
     "fastapi>=0.115.0",
     "uvicorn[standard]>=0.32.0",
-    "python-multipart>=0.0.12",
-    "python-dotenv>=1.0.0",
+    "python-multipart>=0.0.32",
+    "python-dotenv>=1.2.2",
     "opentelemetry-proto>=1.36.0",
     "pyyaml>=6.0",
     "httpx>=0.27.0",
+    # Security floors for CVE-affected transitive dependencies of the default install.
+    # None are imported directly, do not "clean up" as unused. pip does not read
+    # lockfiles, so installs from PyPI resolve from these floors alone.
+    "aiohttp>=3.14.3",
+    "authlib>=1.7.2",
+    "idna>=3.18",
+    "litellm>=1.85.7",
+    "nltk>=3.10.2",
+    "pyasn1>=0.6.4",
+    "requests>=2.34.2",
+    "urllib3>=2.7.0",
 ]
 
 [project.optional-dependencies]
 live = [
-    "mcp>=1.26.0",
+    "mcp>=1.28.1,<2",
     "httpx>=0.27.0",
+    "pyjwt>=2.13.0",
+    "pydantic-settings>=2.15.0",
 ]
 streaming = [
     "opentelemetry-sdk>=1.20.0",

{agentevals_cli-0.9.7 → agentevals_cli-0.9.8}/src/agentevals/streaming/incremental_processor.py

@@ -36,14 +36,18 @@ logger = logging.getLogger(__name__)
 
 
 def _normalize_ts(raw_ts) -> float:
-    """Normalize a nanosecond timestamp (string or int) to seconds."""
+    """Normalize a contemporary Unix timestamp by inferring its unit from magnitude."""
     try:
-        ns = int(raw_ts)
+        timestamp = int(raw_ts)
     except (TypeError, ValueError):
         return 0.0
-    if ns > 1e15:
-        return ns / 1e9
-    return float(ns)
+    if timestamp >= 1e17:
+        return timestamp / 1e9
+    if timestamp >= 1e14:
+        return timestamp / 1e6
+    if timestamp >= 1e11:
+        return timestamp / 1e3
+    return float(timestamp)
 
 
 class IncrementalInvocationExtractor:

{agentevals_cli-0.9.7 → agentevals_cli-0.9.8}/src/agentevals/trace_attrs.py

@@ -81,3 +81,6 @@ ADK_INVOCATION_ID = "gcp.vertex.agent.invocation_id"
 GENAI_ATTRIBUTE_ALIASES: dict[str, list[str]] = {
     OTEL_GENAI_PROVIDER_NAME: [OTEL_GENAI_SYSTEM],
 }
+
+# agentevals custom attributes (repository-specific, outside OTel semconv)
+AGENTEVALS_SESSION_ID = "agentevals.session_id"

{agentevals_cli-0.9.7 → agentevals_cli-0.9.8}/src/agentevals/utils/log_enrichment.py

@@ -7,7 +7,7 @@ import logging
 from collections import defaultdict
 
 from ..trace_attrs import (
-    OTEL_GENAI_AGENT_NAME,
+    AGENTEVALS_SESSION_ID,
     OTEL_GENAI_INPUT_MESSAGES,
     OTEL_GENAI_OUTPUT_MESSAGES,
 )
@@ -28,7 +28,7 @@ def enrich_spans_with_logs(spans: list[dict], logs: list[dict], session_id: str
     Args:
         spans: List of OTLP span dictionaries
         logs: List of GenAI log event dictionaries
-        session_id: Optional session ID to add as agent.name attribute
+        session_id: Optional session ID to add as the agentevals.session_id attribute
 
     Returns:
         List of enriched span dictionaries with message attributes added
@@ -88,6 +88,22 @@ def _extract_messages_from_logs(
     return input_messages, output_messages
 
 
+def _append_session_id(attrs: list[dict], session_id: str | None) -> None:
+    """Append the session ID attribute to *attrs* in place, if present.
+
+    Written to the custom ``agentevals.session_id`` attribute rather than
+    ``gen_ai.agent.name`` so real agent names (from the SDK or OTLP) are
+    never overwritten.
+    """
+    if session_id:
+        attrs.append(
+            {
+                "key": AGENTEVALS_SESSION_ID,
+                "value": {"stringValue": session_id},
+            }
+        )
+
+
 def _inject_messages(
     span: dict,
     input_messages: list[dict],
@@ -113,13 +129,7 @@ def _inject_messages(
                 "value": {"stringValue": json.dumps(output_messages)},
             }
         )
-    if session_id:
-        attrs.append(
-            {
-                "key": OTEL_GENAI_AGENT_NAME,
-                "value": {"stringValue": session_id},
-            }
-        )
+    _append_session_id(attrs, session_id)
 
     return span_copy
 
@@ -148,12 +158,7 @@ def _enrich_per_span(
             span_copy = span.copy()
             if session_id:
                 attrs = list(span_copy.get("attributes", []))
-                attrs.append(
-                    {
-                        "key": OTEL_GENAI_AGENT_NAME,
-                        "value": {"stringValue": session_id},
-                    }
-                )
+                _append_session_id(attrs, session_id)
                 span_copy["attributes"] = attrs
             enriched.append(span_copy)
 

agentevals_cli-0.9.8/tests/test_incremental_processor.py

@@ -0,0 +1,26 @@
+import pytest
+
+from agentevals.streaming.incremental_processor import _normalize_ts
+
+
+@pytest.mark.parametrize(
+    ("raw_ts", "expected"),
+    [
+        (1770000000, 1770000000.0),  # seconds
+        (1770000000000, 1770000000.0),  # milliseconds
+        (1770000000000000, 1770000000.0),  # microseconds
+        (1770000000000000000, 1770000000.0),  # nanoseconds
+    ],
+)
+def test_normalizes_timestamp_units(raw_ts, expected):
+    assert _normalize_ts(raw_ts) == expected
+
+
+def test_normalizes_numeric_string():
+    assert _normalize_ts("1770000000000000") == 1770000000.0
+
+
+@pytest.mark.parametrize("raw_ts", [None, "invalid"])
+def test_invalid_timestamp_returns_zero(raw_ts):
+    # Preserve the existing fallback for missing or malformed timestamps.
+    assert _normalize_ts(raw_ts) == 0.0

{agentevals_cli-0.9.7 → agentevals_cli-0.9.8}/tests/test_log_enrichment.py

@@ -84,8 +84,8 @@ class TestBroadcastEnrichment:
         logs = [_make_log("gen_ai.user.message", {"content": "hi"})]
         result = enrich_spans_with_logs(spans, logs, session_id="my-session")
 
-        agent_name = _get_injected_attr(result[0], "gen_ai.agent.name", parse_json=False)
-        assert agent_name == "my-session"
+        session_attr = _get_injected_attr(result[0], "agentevals.session_id", parse_json=False)
+        assert session_attr == "my-session"
 
     def test_choice_extracts_nested_content(self):
         spans = [_make_span()]
@@ -139,6 +139,23 @@ class TestPerSpanEnrichment:
         msgs_s2 = _get_injected_attr(result[1], "gen_ai.input.messages")
         assert msgs_s2 == [{"role": "user", "content": "question 2"}]
 
+    def test_agent_name_preserved_with_session_id(self):
+        """Regression: real gen_ai.agent.name attr must survive enrichment,
+        session_id lands in its own attr, doesn't clobber it."""
+        span = _make_span(
+            "s1",
+            attrs=[{"key": "gen_ai.agent.name", "value": {"stringValue": "roll_die_agent"}}],
+        )
+        logs = [_make_log("gen_ai.user.message", {"content": "hi"}, span_id="s1")]
+        result = enrich_spans_with_logs([span], logs, session_id="sess-123")
+
+        agent_name_attrs = [a for a in result[0]["attributes"] if a["key"] == "gen_ai.agent.name"]
+        assert len(agent_name_attrs) == 1, f"expected exactly one gen_ai.agent.name attr, found {len(agent_name_attrs)}"
+        assert agent_name_attrs[0]["value"]["stringValue"] == "roll_die_agent"
+
+        session_attr = _get_injected_attr(result[0], "agentevals.session_id", parse_json=False)
+        assert session_attr == "sess-123"
+
     def test_span_without_logs_not_enriched(self):
         spans = [_make_span("s1"), _make_span("s2")]
         logs = [
@@ -156,8 +173,8 @@ class TestPerSpanEnrichment:
         ]
         result = enrich_spans_with_logs(spans, logs, session_id="test")
 
-        assert _get_injected_attr(result[0], "gen_ai.agent.name", parse_json=False) == "test"
-        assert _get_injected_attr(result[1], "gen_ai.agent.name", parse_json=False) == "test"
+        assert _get_injected_attr(result[0], "agentevals.session_id", parse_json=False) == "test"
+        assert _get_injected_attr(result[1], "agentevals.session_id", parse_json=False) == "test"
 
     def test_tool_calls_in_assistant_message(self):
         spans = [_make_span("s1")]

{agentevals_cli-0.9.7 → agentevals_cli-0.9.8}/tests/test_mcp_server.py

@@ -400,5 +400,7 @@ async def test_evaluate_sessions_posts_builtin_evaluator_overrides(monkeypatch):
             "threshold": 0.9,
             "judgeModel": "gemini-2.5-flash",
             "trajectoryMatchType": "IN_ORDER",
+            "credentialRef": None,
+            "judgeBaseUrl": None,
         }
     ]