gooddata-eval 1.76.1.dev2__tar.gz → 1.76.1.dev3__tar.gz

{gooddata_eval-1.76.1.dev2 → gooddata_eval-1.76.1.dev3}/AGENTS.md

@@ -156,6 +156,7 @@ Run this before guessing a `--model` string.
 | `GD_EVAL_AGENT_ID` | which agent to drive, same as `--agent-id` |
 | `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` |
 | `GOODDATA_EVAL_CHAT_*` | SSE retry, backoff and timeout knobs |
+| `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary runs are not labelled |
 | `GD_EVAL_TIMERS` | same as `--timers` |
 
 A gitignored `.env` at the repo root is the normal place for these; load it with

{gooddata_eval-1.76.1.dev2 → gooddata_eval-1.76.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.76.1.dev2
+Version: 1.76.1.dev3
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.76.1.dev2
+Requires-Dist: gooddata-sdk~=1.76.1.dev3
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15

{gooddata_eval-1.76.1.dev2 → gooddata_eval-1.76.1.dev3}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.76.1.dev2"
+version = "1.76.1.dev3"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.76.1.dev2",
+    "gooddata-sdk~=1.76.1.dev3",
     "httpx>=0.27,<1.0",
     "jsonpatch>=1.33,<2.0",
     "orjson>=3.9.15,<4.0.0",

{gooddata_eval-1.76.1.dev2 → gooddata_eval-1.76.1.dev3}/src/gooddata_eval/core/chat/sse_client.py

@@ -13,12 +13,15 @@ module is the single place to swap — the runner only depends on the ChatBacken
 protocol, not on this class.
 """
 
+import functools
 import json
 import logging
 import os
+import re
 import time
 from dataclasses import dataclass, field
 from typing import Any, Callable, Iterable, TypeVar
+from urllib.parse import quote
 
 import httpx
 
@@ -135,6 +138,45 @@ _TURN_TIMEOUT_S = _float_env("GOODDATA_EVAL_CHAT_TURN_TIMEOUT_S", 0.0)
 # 4-turn conversation run to 4x the budget, which is not what a user would sit through.
 _ITEM_TIMEOUT_S = _float_env("GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S", 0.0)
 
+# Comma-separated `key=value` labels, e.g. `model_version=gpt-5.2,github_run_id=42`, sent as
+# W3C baggage on every chat request. gen-ai's Langfuse span processor copies `langfuse_*`
+# baggage onto each span it starts, so every observation of the conversation -- the cost-bearing
+# generations, retries and the title trace included -- carries them from creation. Langfuse v4
+# cannot add them to an observation afterwards. A value cannot contain a comma, and a key must be
+# a plain token (letters, digits, `_`, `-`); any other key is skipped with a warning, because httpx
+# refuses a non-ASCII header and gen-ai's OTel propagator drops or rewrites the rest.
+_TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS"
+_TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+")
+
+
+@functools.cache
+def _warn_skipped_trace_label(key: str) -> None:
+    """Warn once per key per process: every ChatClient re-reads the labels."""
+    _log.warning("%s: skipping label with key %r, keys must match [A-Za-z0-9_-]+", _TRACE_LABELS_ENV, key)
+
+
+def _trace_baggage() -> dict[str, str]:
+    """The `baggage` header for the labels in ``GOODDATA_EVAL_TRACE_LABELS``, or none.
+
+    Each label becomes trace metadata, and ``model_version`` -- the combo the report groups
+    on -- also becomes the observation ``version``, the dimension Langfuse groups cost by.
+    Values are percent-encoded: a space or comma would otherwise end the baggage entry.
+    """
+    labels: dict[str, str] = {}
+    for pair in os.getenv(_TRACE_LABELS_ENV, "").split(","):
+        key, sep, value = pair.partition("=")
+        key = key.strip()
+        if not sep or not key:
+            continue
+        if _TRACE_LABEL_KEY.fullmatch(key):
+            labels[key] = value.strip()
+        else:
+            _warn_skipped_trace_label(key)
+    entries = [f"langfuse_metadata_{k}={quote(v, safe='')}" for k, v in labels.items()]
+    if labels.get("model_version"):
+        entries.insert(0, f"langfuse_version={quote(labels['model_version'], safe='')}")
+    return {"baggage": ",".join(entries)} if entries else {}
+
 
 def set_default_turn_timeout(seconds: float | None) -> None:
     """Set the per-turn budget every ChatClient built afterwards inherits.
@@ -470,7 +512,7 @@ class ChatClient:
         the ``enableAiContextSetup`` feature flag is on for the organization.
         """
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
-        self._auth = {"Authorization": f"Bearer {token}"}
+        self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()}
         # 0/None disables the cap. Also lowered onto the read timeout: the wall-clock check
         # fires between events, so a turn that goes silent needs the transport to give up too.
         budget = _TURN_TIMEOUT_S if turn_timeout_s is None else turn_timeout_s

{gooddata_eval-1.76.1.dev2 → gooddata_eval-1.76.1.dev3}/tests/test_sse_client.py

@@ -1,5 +1,8 @@
 # (C) 2026 GoodData Corporation
 import json
+import logging
+from collections.abc import Callable
+from urllib.parse import unquote
 
 import httpx
 import pytest
@@ -1053,3 +1056,77 @@ def test_ask_puts_the_item_attachment_on_the_wire():
     )
     client.ask(item)
     assert captured["body"]["userContext"] == _ATTACHMENT
+
+
+def _baggage_of(requests: list[httpx.Request]) -> list[dict[str, str] | None]:
+    """Each request's `baggage` header as {key: decoded value}, or None when absent."""
+    out: list[dict[str, str] | None] = []
+    for r in requests:
+        raw = r.headers.get("baggage")
+        out.append(None if raw is None else {k: unquote(v) for k, v in (e.split("=", 1) for e in raw.split(","))})
+    return out
+
+
+def _record_requests(requests: list[httpx.Request]) -> Callable[[httpx.Request], httpx.Response]:
+    def handler(request: httpx.Request) -> httpx.Response:
+        requests.append(request)
+        if request.url.path.endswith("/conversations"):
+            return httpx.Response(200, json={"conversationId": "c1"})
+        return httpx.Response(200, content=_OK_SSE)
+
+    return handler
+
+
+def test_trace_labels_ride_every_request_as_langfuse_baggage(monkeypatch: pytest.MonkeyPatch) -> None:
+    """gen-ai's Langfuse span processor copies `langfuse_*` baggage onto every span it starts, so
+    the labels reach the root generation and its cost-bearing children at creation."""
+    monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "model_version=gpt-5.2 openai,github_run_id=42")
+    requests: list[httpx.Request] = []
+    client = _client_with_handler(_record_requests(requests))
+
+    client.send_message(client.create_conversation(), "q")
+
+    expected = {
+        "langfuse_version": "gpt-5.2 openai",
+        "langfuse_metadata_model_version": "gpt-5.2 openai",
+        "langfuse_metadata_github_run_id": "42",
+    }
+    assert _baggage_of(requests) == [expected, expected]
+    # Encoded, not raw: a space or comma in a value would otherwise split or end the entry.
+    assert " " not in requests[0].headers["baggage"]
+
+
+def test_trace_labels_without_a_model_version_set_no_langfuse_version(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", "github_run_id=42")
+    requests: list[httpx.Request] = []
+    _client_with_handler(_record_requests(requests)).create_conversation()
+
+    assert _baggage_of(requests) == [{"langfuse_metadata_github_run_id": "42"}]
+
+
+@pytest.mark.parametrize("key", ["modèle", "run id", "c++"])
+def test_trace_label_with_a_non_token_key_is_skipped_with_one_warning(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, key: str
+) -> None:
+    """httpx refuses a non-ASCII header, gen-ai's OTel propagator drops a key with a space, and
+    it reads `+` as a space; each would otherwise fail the run or mislabel the trace."""
+    monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", f"github_run_id=42,{key}=1")
+    requests: list[httpx.Request] = []
+
+    for _ in range(2):
+        _client_with_handler(_record_requests(requests)).create_conversation()
+
+    assert _baggage_of(requests) == [{"langfuse_metadata_github_run_id": "42"}] * 2
+    assert [r.levelno for r in caplog.records if repr(key) in r.getMessage()] == [logging.WARNING]
+
+
+@pytest.mark.parametrize("raw", [None, "", "no-equals-sign", " , "])
+def test_no_trace_labels_send_no_baggage(monkeypatch: pytest.MonkeyPatch, raw: str | None) -> None:
+    if raw is None:
+        monkeypatch.delenv("GOODDATA_EVAL_TRACE_LABELS", raising=False)
+    else:
+        monkeypatch.setenv("GOODDATA_EVAL_TRACE_LABELS", raw)
+    requests: list[httpx.Request] = []
+    _client_with_handler(_record_requests(requests)).create_conversation()
+
+    assert _baggage_of(requests) == [None]