gooddata-eval 1.78.1.dev3__py3-none-any.whl → 1.78.1.dev4__py3-none-any.whl

gooddata_eval/core/agentic/document_skill.py

@@ -32,7 +32,7 @@ from gooddata_eval.core.agentic._trace_linker import (
 from gooddata_eval.core.agentic.dashboard_skill import _extract_tool_result, _skill_activated
 from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
-from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.config import DocumentFormat, ReasoningEffort, normalize_document_format
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run
 from gooddata_eval.core.langfuse.item_scope import item_scope
 from gooddata_eval.core.models import (
@@ -801,18 +801,24 @@ def run_agentic_document_skill(
     agent_id: str | None = None,
     judge: LLMJudge | None = None,
     user_context: dict | None = None,
+    document_format: DocumentFormat | None = None,
 ) -> AgenticDocumentSummary:
     """Run the document-skill agentic evaluation K times and return a summary.
 
     The narrative judge is built only for a fixture that states a ``narrative``, so an item
     without one needs neither the llm-judge extra nor ``OPENAI_API_KEY``.
 
+    ``document_format`` is sent on every turn of every run, see ``ChatClient``; None leaves
+    the format to the server's ``enablePublisherV2`` feature flag.
+
     Raises:
         ValueError: the fixture is unusable — see ``_validate_expectation`` and
-            ``_validate_user_context``.
+            ``_validate_user_context``, or ``document_format`` is not a format gen-ai
+            accepts. Raised before any chat request.
     """
     _validate_expectation(expected_output)
     _validate_user_context(user_context)
+    document_format = normalize_document_format(document_format)
     if judge is None and _has_narrative(expected_output):
         judge = LLMJudge(_NARRATIVE_EVALUATION_STEPS)
     run_results: list[DocumentRunResult] = []
@@ -823,6 +829,7 @@ def run_agentic_document_skill(
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
         user_context=user_context,
+        document_format=document_format,
     )
 
     try:
@@ -882,20 +889,32 @@ def evaluate_agentic_document_skill(
     gate: EvalGate = DEFAULT_GATE,
     judge: LLMJudge | None = None,
     user_context: dict | None = None,
+    document_format: DocumentFormat | None = None,
 ) -> AgenticEvalOutcome:
     """Run document-skill evaluation, log to Langfuse, and raise on failure.
 
     Returns the best run's outcome on success; on failure the same values are attached to the
     raised ``DocumentSkillAssertionError`` so callers can retrieve them either way.
 
+    ``document_format`` (``aac-v1``/``aac-v2``) asks gen-ai for that document content format
+    on every turn and is recorded as ``document_format`` in the dataset-run metadata; None
+    leaves it to the server's ``enablePublisherV2`` feature flag.
+
     Raises:
         DocumentSkillAssertionError: the gate did not pass.
-        ValueError: the fixture is unusable — see ``_validate_expectation`` and
-            ``_validate_user_context``. Raised before any
-            request, so it means a fixture to fix rather than a result to read.
+        ValueError: the fixture is unusable, see ``_validate_expectation`` and
+            ``_validate_user_context``. Raised before any chat request, so it means a
+            fixture to fix rather than a result to read. Also raised, equally before any
+            request, when the caller passes a ``document_format`` gen-ai does not accept.
         JudgeResponseError: the fixture states a narrative and the judge returned no readable
             verdict for any run -- an item without a result, not K failures.
     """
+    document_format = normalize_document_format(document_format)
+    if document_format is not None:
+        # Metadata only, not part of the run name: it tells runs apart only when their names
+        # already differ, so two runs that share a run_timestamp, dataset and model but not the
+        # format still collide on one dataset-run name.
+        run_metadata_extra = {**(run_metadata_extra or {}), "document_format": document_format}
     identity = RunIdentity(
         host,
         token,
@@ -921,6 +940,7 @@ def evaluate_agentic_document_skill(
             agent_id=agent_id,
             judge=judge,
             user_context=user_context,
+            document_format=document_format,
         )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/chat/sse_client.py

@@ -26,7 +26,12 @@ from urllib.parse import quote
 import httpx
 
 from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, float_env, int_env, retry_transient
-from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
+from gooddata_eval.core.config import (
+    DocumentFormat,
+    ReasoningEffort,
+    normalize_document_format,
+    normalize_reasoning_effort,
+)
 from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for
 from gooddata_eval.core.models import ChatResult, DatasetItem
 from gooddata_eval.core.timing import note_trace_latency
@@ -460,6 +465,7 @@ class ChatClient:
         reasoning_effort: ReasoningEffort | None = None,
         agent_id: str | None = None,
         user_context: dict[str, Any] | None = None,
+        document_format: DocumentFormat | None = None,
     ):
         """Create a chat client bound to one workspace.
 
@@ -476,6 +482,13 @@ class ChatClient:
         back to it: a caller that changes or clears the context per message builds the
         client without one. The server grounds answers in it only while
         the ``enableAiContextSetup`` feature flag is on for the organization.
+
+        ``document_format`` (``aac-v1``/``aac-v2``) is sent as ``options.documentFormat``
+        on every message and selects the document content format gen-ai drafts in for
+        that turn. gen-ai keeps a draft in the format of the turn that wrote it and a
+        turn in the other format cannot continue it, so follow-up and clarification
+        turns carry it too. When None the key is omitted and the server picks the format
+        from the ``enablePublisherV2`` feature flag. Any other value raises ``ValueError``.
         """
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
         self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()}
@@ -494,6 +507,7 @@ class ChatClient:
         self._client = httpx.Client(timeout=http_timeout)
         self._preserve_failed = preserve_failed
         self._reasoning_effort = normalize_reasoning_effort(reasoning_effort)
+        self._document_format = normalize_document_format(document_format)
         self._agent_id = agent_id
         self._user_context = user_context
         # Run index of each conversation this client has sent to, in first-send order: every
@@ -531,8 +545,13 @@ class ChatClient:
         join_headers, joined = self._join_item_trace(conversation_id)
         headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json", **join_headers}
         body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
+        options: dict[str, str] = {}
         if self._reasoning_effort is not None:
-            body["options"] = {"reasoningEffort": self._reasoning_effort}
+            options["reasoningEffort"] = self._reasoning_effort
+        if self._document_format is not None:
+            options["documentFormat"] = self._document_format
+        if options:
+            body["options"] = options
         # A per-call context overrides the client's. Only sent when there is one: gen-ai
         # accepts an explicit null, so assigning unconditionally would quietly change every
         # request that has no attachment.

gooddata_eval/core/config.py

@@ -33,6 +33,9 @@ def judge_model() -> str:
 ReasoningEffort = Literal["LOW", "MEDIUM", "HIGH"]
 """Effort values the AI chat endpoint accepts, uppercase as the server enum requires."""
 
+DocumentFormat = Literal["aac-v1", "aac-v2"]
+"""Document content formats the AI chat endpoint accepts as ``options.documentFormat``."""
+
 EvalGate = Literal["any", "power"]
 """`any` = pass@K, `power` = pass^K. Behaviour is in core/agentic/_gate.py; the type lives here
 because putting it there would make config import the agentic package, whose __init__ imports
@@ -73,6 +76,22 @@ def normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None:
     return cast("ReasoningEffort", candidate)
 
 
+def normalize_document_format(value: str | None) -> DocumentFormat | None:
+    """Canonical document format, or None when unset.
+
+    gen-ai rejects any value outside `DocumentFormat`, so an unknown value raises here,
+    before a run sends its first request.
+    """
+    if value is None:
+        return None
+    candidate = value.strip().lower()
+    if not candidate:
+        return None
+    if candidate not in get_args(DocumentFormat):
+        raise ValueError(f"Invalid document format {value!r}; expected one of {', '.join(get_args(DocumentFormat))}.")
+    return cast("DocumentFormat", candidate)
+
+
 @dataclass
 class RunConfig:
     host: str

{gooddata_eval-1.78.1.dev3.dist-info → gooddata_eval-1.78.1.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.78.1.dev3
+Version: 1.78.1.dev4
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.78.1.dev3
+Requires-Dist: gooddata-sdk~=1.78.1.dev4
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15

{gooddata_eval-1.78.1.dev3.dist-info → gooddata_eval-1.78.1.dev4.dist-info}/RECORD

@@ -6,7 +6,7 @@ gooddata_eval/cli/main.py,sha256=ZZv9pViwV4h21LPMrBOUCH99-oyM9dd2AFpGj9NWEJI,344
 gooddata_eval/core/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/_output.py,sha256=sFbNtKg1lx431jpb15jISLlOXGTZ-r_KodfaTVyka6I,580
 gooddata_eval/core/_retry.py,sha256=86ZPWu6ahCYC2GLvcbwjHTtj2RrGPOuxIFlCbtmNpvw,2990
-gooddata_eval/core/config.py,sha256=HegNbWb4csXXGaOZiFkplhsr8FmVK7OJL_lDKne1Sdg,4080
+gooddata_eval/core/config.py,sha256=hla5t294uHC5Q9bzrhPjsdkwEcetLsEH-GqGAhs94b4,4823
 gooddata_eval/core/connection.py,sha256=mh-Do5-Ti0-DXXikfCIH10eTTUbKuZUAS_sLoPNDJ5I,1229
 gooddata_eval/core/granularity.py,sha256=CbniwgxtVj0oyf41UjeeZbBzOpeisSBlAfdkL79e6O0,3932
 gooddata_eval/core/interaction.py,sha256=PUaJ1zdMwXBKjoNXNFXVgt_b8_i0LWdbMZndEQ3F6UM,2093
@@ -27,7 +27,7 @@ gooddata_eval/core/agentic/alert_skill.py,sha256=Me7YrAAGWG6iqzJR18JdhVsri4LG4J1
 gooddata_eval/core/agentic/anomaly_detection.py,sha256=H_rKx6fCX3S11HzGAQGhy092YMmGj4foFBmrx6U-LHE,27701
 gooddata_eval/core/agentic/conversation.py,sha256=_OLlST1twnSPnOQGfuuFCFKu92uwLVh2YBBA_nOW-5w,60004
 gooddata_eval/core/agentic/dashboard_skill.py,sha256=PnFq1JjaMpAjQkzgklRkEFFKEs4WWhORikDVw5qjbaE,84630
-gooddata_eval/core/agentic/document_skill.py,sha256=Z9jzWhOjaPmQze1IeDrc08pg0XB31_NTWBRQkxXf178,43859
+gooddata_eval/core/agentic/document_skill.py,sha256=Uukqc6voa3lXbcSVgfzQyPH_kEFu3jrbrN9EDIMIuYA,45231
 gooddata_eval/core/agentic/general_question.py,sha256=pOVYlJF9eKZwTqdPVsrH4X1OC-tXI7CMCFDdPNdBIMw,17758
 gooddata_eval/core/agentic/guardrail.py,sha256=1Yz5QVVWo5GCOA8OKSy5efeysaaXB3bXcr_BT-Qu8tY,16398
 gooddata_eval/core/agentic/kda_skill.py,sha256=2mav9itclavID28U4fORSR9p-iVTrddMyTHLKh8HxdE,26154
@@ -38,7 +38,7 @@ gooddata_eval/core/agentic/visualization.py,sha256=o4RVS-nRCVbVXjbE_jHuGzgW8x3hR
 gooddata_eval/core/agentic/what_if.py,sha256=v5u9Yl45QGpVD2IHvPhVW_XOixI10IK4N-5_zP9Yymw,26365
 gooddata_eval/core/chat/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/chat/render.py,sha256=CqsKN8__DlaiJj_bmcPZ1poc5LfUqtz7EzKRDbks4gU,2082
-gooddata_eval/core/chat/sse_client.py,sha256=UGmghNUQHGM5NMBtAAJj6Z9jQpgs0ROfqH9egw3Tgvo,31006
+gooddata_eval/core/chat/sse_client.py,sha256=yGUbWg7gr7ycVfEpvpmGUCIQ38nW0jiMmuJB9_va4UY,31921
 gooddata_eval/core/dataset/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/dataset/from_insights.py,sha256=2eYh_84070t2BMeW2XPLAwzrJs7vKbFYMeCppvkDiEo,40880
 gooddata_eval/core/dataset/langfuse_source.py,sha256=II-6fenEPhdyfckrL0iVdJpavjv67wUvy7OIuvHntrg,8697
@@ -77,8 +77,8 @@ gooddata_eval/core/summary/http_client.py,sha256=Umae42fiiJQ0e4hv63NRROHYvcFAGgX
 gooddata_eval/core/takeaway/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/takeaway/evaluate.py,sha256=ei7DLSlG_Ove_R551Ofdw8WISf178XknU58jBWxB_is,11585
 gooddata_eval/core/takeaway/http_client.py,sha256=yc3FNPArjfcSdAE7naRf7g_aW7BbJhwsfSEvAFd67Ik,4141
-gooddata_eval-1.78.1.dev3.dist-info/METADATA,sha256=mEAMzHCMbrgw87nzERRGspV50OszmLr0zRgcY4tMVxg,60351
-gooddata_eval-1.78.1.dev3.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-gooddata_eval-1.78.1.dev3.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
-gooddata_eval-1.78.1.dev3.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
-gooddata_eval-1.78.1.dev3.dist-info/RECORD,,
+gooddata_eval-1.78.1.dev4.dist-info/METADATA,sha256=uQZy3GjLJ3V1W9B9RR9kX6fLzD5-Var8lEQApCzH3iA,60351
+gooddata_eval-1.78.1.dev4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+gooddata_eval-1.78.1.dev4.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
+gooddata_eval-1.78.1.dev4.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
+gooddata_eval-1.78.1.dev4.dist-info/RECORD,,