gooddata-eval 1.78.1.dev3__tar.gz → 1.78.1.dev4__tar.gz

{gooddata_eval-1.78.1.dev3 → gooddata_eval-1.78.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.78.1.dev3
+Version: 1.78.1.dev4
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.78.1.dev3
+Requires-Dist: gooddata-sdk~=1.78.1.dev4
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15

{gooddata_eval-1.78.1.dev3 → gooddata_eval-1.78.1.dev4}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.78.1.dev3"
+version = "1.78.1.dev4"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.78.1.dev3",
+    "gooddata-sdk~=1.78.1.dev4",
     "httpx>=0.27,<1.0",
     "jsonpatch>=1.33,<2.0",
     "orjson>=3.9.15,<4.0.0",

{gooddata_eval-1.78.1.dev3 → gooddata_eval-1.78.1.dev4}/src/gooddata_eval/core/agentic/document_skill.py

@@ -32,7 +32,7 @@ from gooddata_eval.core.agentic._trace_linker import (
 from gooddata_eval.core.agentic.dashboard_skill import _extract_tool_result, _skill_activated
 from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
-from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.config import DocumentFormat, ReasoningEffort, normalize_document_format
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run
 from gooddata_eval.core.langfuse.item_scope import item_scope
 from gooddata_eval.core.models import (
@@ -801,18 +801,24 @@ def run_agentic_document_skill(
     agent_id: str | None = None,
     judge: LLMJudge | None = None,
     user_context: dict | None = None,
+    document_format: DocumentFormat | None = None,
 ) -> AgenticDocumentSummary:
     """Run the document-skill agentic evaluation K times and return a summary.
 
     The narrative judge is built only for a fixture that states a ``narrative``, so an item
     without one needs neither the llm-judge extra nor ``OPENAI_API_KEY``.
 
+    ``document_format`` is sent on every turn of every run, see ``ChatClient``; None leaves
+    the format to the server's ``enablePublisherV2`` feature flag.
+
     Raises:
         ValueError: the fixture is unusable — see ``_validate_expectation`` and
-            ``_validate_user_context``.
+            ``_validate_user_context``, or ``document_format`` is not a format gen-ai
+            accepts. Raised before any chat request.
     """
     _validate_expectation(expected_output)
     _validate_user_context(user_context)
+    document_format = normalize_document_format(document_format)
     if judge is None and _has_narrative(expected_output):
         judge = LLMJudge(_NARRATIVE_EVALUATION_STEPS)
     run_results: list[DocumentRunResult] = []
@@ -823,6 +829,7 @@ def run_agentic_document_skill(
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
         user_context=user_context,
+        document_format=document_format,
     )
 
     try:
@@ -882,20 +889,32 @@ def evaluate_agentic_document_skill(
     gate: EvalGate = DEFAULT_GATE,
     judge: LLMJudge | None = None,
     user_context: dict | None = None,
+    document_format: DocumentFormat | None = None,
 ) -> AgenticEvalOutcome:
     """Run document-skill evaluation, log to Langfuse, and raise on failure.
 
     Returns the best run's outcome on success; on failure the same values are attached to the
     raised ``DocumentSkillAssertionError`` so callers can retrieve them either way.
 
+    ``document_format`` (``aac-v1``/``aac-v2``) asks gen-ai for that document content format
+    on every turn and is recorded as ``document_format`` in the dataset-run metadata; None
+    leaves it to the server's ``enablePublisherV2`` feature flag.
+
     Raises:
         DocumentSkillAssertionError: the gate did not pass.
-        ValueError: the fixture is unusable — see ``_validate_expectation`` and
-            ``_validate_user_context``. Raised before any
-            request, so it means a fixture to fix rather than a result to read.
+        ValueError: the fixture is unusable, see ``_validate_expectation`` and
+            ``_validate_user_context``. Raised before any chat request, so it means a
+            fixture to fix rather than a result to read. Also raised, equally before any
+            request, when the caller passes a ``document_format`` gen-ai does not accept.
         JudgeResponseError: the fixture states a narrative and the judge returned no readable
             verdict for any run -- an item without a result, not K failures.
     """
+    document_format = normalize_document_format(document_format)
+    if document_format is not None:
+        # Metadata only, not part of the run name: it tells runs apart only when their names
+        # already differ, so two runs that share a run_timestamp, dataset and model but not the
+        # format still collide on one dataset-run name.
+        run_metadata_extra = {**(run_metadata_extra or {}), "document_format": document_format}
     identity = RunIdentity(
         host,
         token,
@@ -921,6 +940,7 @@ def evaluate_agentic_document_skill(
             agent_id=agent_id,
             judge=judge,
             user_context=user_context,
+            document_format=document_format,
         )
 
     if langfuse is not None and dataset_item_id:

{gooddata_eval-1.78.1.dev3 → gooddata_eval-1.78.1.dev4}/src/gooddata_eval/core/chat/sse_client.py

@@ -26,7 +26,12 @@ from urllib.parse import quote
 import httpx
 
 from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, float_env, int_env, retry_transient
-from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
+from gooddata_eval.core.config import (
+    DocumentFormat,
+    ReasoningEffort,
+    normalize_document_format,
+    normalize_reasoning_effort,
+)
 from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for
 from gooddata_eval.core.models import ChatResult, DatasetItem
 from gooddata_eval.core.timing import note_trace_latency
@@ -460,6 +465,7 @@ class ChatClient:
         reasoning_effort: ReasoningEffort | None = None,
         agent_id: str | None = None,
         user_context: dict[str, Any] | None = None,
+        document_format: DocumentFormat | None = None,
     ):
         """Create a chat client bound to one workspace.
 
@@ -476,6 +482,13 @@ class ChatClient:
         back to it: a caller that changes or clears the context per message builds the
         client without one. The server grounds answers in it only while
         the ``enableAiContextSetup`` feature flag is on for the organization.
+
+        ``document_format`` (``aac-v1``/``aac-v2``) is sent as ``options.documentFormat``
+        on every message and selects the document content format gen-ai drafts in for
+        that turn. gen-ai keeps a draft in the format of the turn that wrote it and a
+        turn in the other format cannot continue it, so follow-up and clarification
+        turns carry it too. When None the key is omitted and the server picks the format
+        from the ``enablePublisherV2`` feature flag. Any other value raises ``ValueError``.
         """
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
         self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()}
@@ -494,6 +507,7 @@ class ChatClient:
         self._client = httpx.Client(timeout=http_timeout)
         self._preserve_failed = preserve_failed
         self._reasoning_effort = normalize_reasoning_effort(reasoning_effort)
+        self._document_format = normalize_document_format(document_format)
         self._agent_id = agent_id
         self._user_context = user_context
         # Run index of each conversation this client has sent to, in first-send order: every
@@ -531,8 +545,13 @@ class ChatClient:
         join_headers, joined = self._join_item_trace(conversation_id)
         headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json", **join_headers}
         body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
+        options: dict[str, str] = {}
         if self._reasoning_effort is not None:
-            body["options"] = {"reasoningEffort": self._reasoning_effort}
+            options["reasoningEffort"] = self._reasoning_effort
+        if self._document_format is not None:
+            options["documentFormat"] = self._document_format
+        if options:
+            body["options"] = options
         # A per-call context overrides the client's. Only sent when there is one: gen-ai
         # accepts an explicit null, so assigning unconditionally would quietly change every
         # request that has no attachment.

{gooddata_eval-1.78.1.dev3 → gooddata_eval-1.78.1.dev4}/src/gooddata_eval/core/config.py

@@ -33,6 +33,9 @@ def judge_model() -> str:
 ReasoningEffort = Literal["LOW", "MEDIUM", "HIGH"]
 """Effort values the AI chat endpoint accepts, uppercase as the server enum requires."""
 
+DocumentFormat = Literal["aac-v1", "aac-v2"]
+"""Document content formats the AI chat endpoint accepts as ``options.documentFormat``."""
+
 EvalGate = Literal["any", "power"]
 """`any` = pass@K, `power` = pass^K. Behaviour is in core/agentic/_gate.py; the type lives here
 because putting it there would make config import the agentic package, whose __init__ imports
@@ -73,6 +76,22 @@ def normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None:
     return cast("ReasoningEffort", candidate)
 
 
+def normalize_document_format(value: str | None) -> DocumentFormat | None:
+    """Canonical document format, or None when unset.
+
+    gen-ai rejects any value outside `DocumentFormat`, so an unknown value raises here,
+    before a run sends its first request.
+    """
+    if value is None:
+        return None
+    candidate = value.strip().lower()
+    if not candidate:
+        return None
+    if candidate not in get_args(DocumentFormat):
+        raise ValueError(f"Invalid document format {value!r}; expected one of {', '.join(get_args(DocumentFormat))}.")
+    return cast("DocumentFormat", candidate)
+
+
 @dataclass
 class RunConfig:
     host: str

{gooddata_eval-1.78.1.dev3 → gooddata_eval-1.78.1.dev4}/tests/test_agentic_document_skill.py

@@ -536,6 +536,74 @@ def test_a_user_context_with_view_report_fails_before_any_request(monkeypatch: p
     assert client.sent == []
 
 
+def test_the_document_format_reaches_the_chat_client(monkeypatch: pytest.MonkeyPatch) -> None:
+    client = _ScriptedChatClient([_drafting_turn()])
+    built: list[dict[str, Any]] = []
+
+    def _build(**kwargs: Any) -> _ScriptedChatClient:
+        built.append(kwargs)
+        return client
+
+    monkeypatch.setattr(document_skill, "ChatClient", _build)
+    evaluate_agentic_document_skill("https://h", "tok", "ws", "Create a document", {}, document_format="aac-v2")
+    assert [kwargs["document_format"] for kwargs in built] == ["aac-v2"]
+
+
+def test_an_unknown_document_format_fails_before_any_request(monkeypatch: pytest.MonkeyPatch) -> None:
+    client = _ScriptedChatClient([_drafting_turn()])
+    _install_client(monkeypatch, client)
+    with pytest.raises(ValueError, match="Invalid document format 'v2'"):
+        evaluate_agentic_document_skill("https://h", "tok", "ws", "Create a document", {}, document_format="v2")
+    assert client.created == client.sent == []
+
+
+def test_an_unknown_document_format_fails_before_the_narrative_judge_is_built(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    client = _ScriptedChatClient([_drafting_turn()])
+    _install_client(monkeypatch, client)
+    with pytest.raises(ValueError, match="Invalid document format 'v2'"):
+        run_agentic_document_skill("https://h", "tok", "ws", "q", {"narrative": _NARRATIVE}, document_format="v2")
+    assert client.created == client.sent == []
+
+
+@pytest.mark.parametrize(
+    ("document_format", "extra", "expected"),
+    [
+        ("AAC-V2", {"tags": ["ci"]}, {"tags": ["ci"], "document_format": "aac-v2"}),
+        (None, {"tags": ["ci"]}, {"tags": ["ci"]}),
+        (None, None, None),
+    ],
+)
+def test_the_document_format_is_recorded_in_the_run_metadata(
+    monkeypatch: pytest.MonkeyPatch,
+    document_format: str | None,
+    extra: dict[str, Any] | None,
+    expected: dict[str, Any] | None,
+) -> None:
+    _install_client(monkeypatch, _ScriptedChatClient([_drafting_turn()]))
+    identities: list[Any] = []
+    monkeypatch.setattr(
+        document_skill, "submit_trace_scoring", lambda _link, identity, **_kw: identities.append(identity)
+    )
+    try:
+        evaluate_agentic_document_skill(
+            "https://h",
+            "tok",
+            "ws",
+            "Create a document",
+            {},
+            langfuse=object(),
+            dataset_item_id="item-1",
+            run_metadata_extra=extra,
+            document_format=document_format,
+        )
+    except DocumentSkillAssertionError:
+        pass  # the identity is handed to scoring before the gate raises
+    assert [identity.run_metadata_extra for identity in identities] == [expected]
+
+
 # ── fixture validation ──────────────────────────────────────────────────────
 
 

{gooddata_eval-1.78.1.dev3 → gooddata_eval-1.78.1.dev4}/tests/test_sse_client.py

@@ -2,6 +2,7 @@
 import json
 import logging
 from collections.abc import Callable
+from typing import Any
 from urllib.parse import unquote
 
 import httpx
@@ -951,12 +952,23 @@ def test_ask_attaches_conversation_id_to_exception(monkeypatch):
     assert ei.value.conversation_id == "conv-exc"
 
 
-def _capture_body_client(captured, *, reasoning_effort=None):
-    def handler(request):
+def _capture_body_client(
+    captured: list[dict[str, Any]],
+    *,
+    reasoning_effort: str | None = None,
+    document_format: str | None = None,
+) -> ChatClient:
+    def handler(request: httpx.Request) -> httpx.Response:
         captured.append(json.loads(request.content))
         return httpx.Response(200, content=_OK_SSE)
 
-    client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort=reasoning_effort)
+    client = ChatClient(
+        host="https://example.invalid",
+        token="t",
+        workspace_id="w",
+        reasoning_effort=reasoning_effort,
+        document_format=document_format,
+    )
     client._client = httpx.Client(transport=httpx.MockTransport(handler))
     return client
 
@@ -1026,6 +1038,43 @@ def test_invalid_reasoning_effort_fails_at_construction():
         ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort="maximum")
 
 
+@pytest.mark.parametrize("document_format", ["aac-v1", "aac-v2"])
+def test_send_message_sends_document_format_on_every_message(document_format: str) -> None:
+    """gen-ai keeps a draft in the format of the turn that wrote it, so follow-ups need it too."""
+    captured: list[dict[str, Any]] = []
+    client = _capture_body_client(captured, document_format=document_format)
+    client.send_message("conv", "first")
+    client.send_message("conv", "follow-up")
+    assert [c["options"] for c in captured] == [{"documentFormat": document_format}] * 2
+
+
+def test_send_message_sends_document_format_with_reasoning_effort() -> None:
+    captured: list[dict[str, Any]] = []
+    _capture_body_client(captured, reasoning_effort="LOW", document_format="aac-v2").send_message("conv", "q")
+    assert captured[0]["options"] == {"reasoningEffort": "LOW", "documentFormat": "aac-v2"}
+
+
+@pytest.mark.parametrize(("given", "sent"), [("AAC-V2", "aac-v2"), ("  aac-v1  ", "aac-v1")])
+def test_send_message_normalizes_document_format(given: str, sent: str) -> None:
+    captured: list[dict[str, Any]] = []
+    _capture_body_client(captured, document_format=given).send_message("conv", "q")
+    assert captured[0]["options"] == {"documentFormat": sent}
+
+
+@pytest.mark.parametrize("blank", ["", "   "])
+def test_blank_document_format_is_treated_as_unset(blank: str) -> None:
+    captured: list[dict[str, Any]] = []
+    _capture_body_client(captured, document_format=blank).send_message("conv", "q")
+    assert "options" not in captured[0]
+
+
+@pytest.mark.parametrize("bad", ["v2", "aac-v3", "aac_v2"])
+def test_invalid_document_format_fails_at_construction(bad: str) -> None:
+    """gen-ai rejects any other value; fail before the first request instead."""
+    with pytest.raises(ValueError, match="Invalid document format"):
+        ChatClient(host="https://example.invalid", token="t", workspace_id="w", document_format=bad)
+
+
 def test_turn_timeout_aborts_a_streaming_turn_and_is_not_retried(monkeypatch):
     """A chatty-but-slow agent must be cut off: httpx's per-read timeout never fires
     for one, so only the wall-clock budget bounds the item."""