gooddata-eval 1.78.1.dev2__py3-none-any.whl → 1.78.1.dev4__py3-none-any.whl

gooddata_eval/core/agentic/_catalog.py

@@ -67,6 +67,12 @@ class CatalogMetricAlert:
     """Expected group-by attributes; ``None`` means the fixture states no expectation."""
     granularity: AnomalyDetectionGranularity | None = None
     """Detection interval for an ANOMALY alert (DAY/WEEK/MONTH/...). Not a date filter."""
+    comparison: dict[str, str | int | float | None] | None = None
+    """Expected period comparison as ``create_metric_alert`` argument names; ``None`` means unasserted."""
+    title: str | None = None
+    """Alert name the user asked for; ``None`` means unasserted."""
+    dashboard_id: str | None = None
+    """Dashboard the alert must be bound to; ``None`` means unasserted."""
 
     @classmethod
     def from_dict(cls, d: dict) -> CatalogMetricAlert:
@@ -87,4 +93,7 @@ class CatalogMetricAlert:
             filters=d.get("filters"),
             attributes=d.get("attributes"),
             granularity=AnomalyDetectionGranularity.parse(d.get("granularity")),
+            comparison=d.get("comparison"),
+            title=d.get("title"),
+            dashboard_id=d.get("dashboard_id"),
         )

gooddata_eval/core/agentic/alert_skill.py

@@ -58,6 +58,37 @@ _TRIGGER_DISPLAY_TO_API = {"Every time": "ALWAYS", "One time": "ONCE"}
 _ALWAYS_TRIGGER_VALUES = {"Every time", "ALWAYS", "not specified"}
 _RELATIVE_OPERATORS = ("INCREASES_BY", "DECREASES_BY", "CHANGES_BY")
 
+# Fixture spelling of each period-comparison property -> the create_metric_alert argument.
+_COMPARISON_ARGS = {
+    "kind": "comparison_kind",
+    "periods_ago": "comparison_periods_ago",
+    "date_dataset_id": "comparison_date_dataset_id",
+    "date_attribute_id": "comparison_date_attribute_id",
+    "arithmetic_operator": "arithmetic_operator",
+}
+# Compared verbatim: object ids are case-sensitive, unlike the enum-valued properties.
+_COMPARISON_ID_ARGS = ("comparison_date_dataset_id", "comparison_date_attribute_id")
+# What create_metric_alert applies when the argument is left out, per its own contract. A
+# metric-to-metric comparison has no periods, so periods_ago defaults only for a period kind.
+_COMPARISON_ARG_DEFAULTS: dict[str, object] = {"comparison_periods_ago": 1, "arithmetic_operator": "DIFFERENCE"}
+_PERIOD_COMPARISON_KINDS = ("PREVIOUS_PERIOD", "SAME_PERIOD_PRIOR_YEAR")
+_NO_COMPARISON_MARKERS = ("none", "no comparison")
+
+# AlertEvaluation checks, in report order. Every run is scored on the first group; the second only
+# when the item states the matching expectation.
+_ALERT_CHECKS = (
+    "alert_created",
+    "operator_correct",
+    "threshold_correct",
+    "trigger_correct",
+    "filters_correct",
+    "metric_correct",
+    "recipients_correct",
+    "attributes_correct",
+    "granularity_correct",
+)
+_CONDITIONAL_ALERT_CHECKS = ("comparison_correct", "title_correct", "dashboard_correct")
+
 # Expected-recipient spellings that mean the owner of the API token.
 _CURRENT_USER_RECIPIENT = "me"
 _CURRENT_USER_MARKERS = {"me", "myself", "current user", "current_user"}
@@ -94,6 +125,29 @@ def _parse_recipients(recipients_str: str | None) -> list[str] | None:
     return [r.strip() for r in recipients_str.replace(";", ",").split(",") if r.strip()]
 
 
+def _recipient_list(value: object, *, split_bare: bool) -> list[str]:
+    """A recipient argument as a list, decoded the way gen-ai does before calling the server.
+
+    The tool schema declares both recipient fields ``anyOf: [array of string, string, null]``,
+    and some models send the list JSON-encoded as a string (``'["admin"]'``). Empty or
+    ``"null"`` means none; any other string that is not a JSON list of strings is a bare
+    value -- one user id, or with ``split_bare`` a comma/semicolon-separated email list
+    (gen-ai forwards a bare string as one entry; the split is this checker's own leniency).
+    A non-string list entry is kept as its ``str()`` so it compares unequal instead of raising.
+    """
+    if isinstance(value, list):
+        return [str(item) for item in value]
+    if not isinstance(value, str) or value.strip() in ("", "null"):
+        return []
+    try:
+        parsed = json.loads(value)
+    except ValueError:
+        parsed = None
+    if isinstance(parsed, list) and all(isinstance(item, str) for item in parsed):
+        return parsed
+    return (_parse_recipients(value) or []) if split_bare else [value]
+
+
 def _deep_subset(expected: object, actual: object) -> bool:
     """Return True if expected is a recursive subset of actual."""
     if isinstance(expected, dict) and isinstance(actual, dict):
@@ -234,6 +288,66 @@ def _check_granularity(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     return str(actual).strip().upper() == expected.granularity.value
 
 
+def _canonical_comparison_value(arg: str, value: object) -> str | int | float | None:
+    """Fold one comparison value so fixture and tool-argument spellings compare equal."""
+    if value is None or str(value).strip() == "":
+        return None
+    if arg == "comparison_periods_ago":
+        return _to_number(value)
+    if arg in _COMPARISON_ID_ARGS:
+        return str(value).strip()
+    return str(value).strip().upper()
+
+
+def _check_comparison(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+    """Compare only the comparison properties the fixture states.
+
+    ``None`` means unasserted. A stated ``comparison_kind`` of ``None`` asserts the alert has
+    no comparison at all. ``comparison_kind`` is read as ``create_metric_alert`` resolves it:
+    left out next to a ``compare_metric_id`` it is METRIC. Any other argument the agent left out
+    on an alert that does compare counts as the value ``create_metric_alert`` applies for it, so
+    omitting ``comparison_periods_ago`` matches 1 on a period comparison; a METRIC comparison has
+    no periods, and with no comparison there is nothing to default.
+    The comparison period itself (DAY, MONTH, ...) is not checked here: it is the granularity
+    of the comparison date filter, which ``filters`` asserts.
+    """
+    if expected.comparison is None:
+        return True
+    kind = actual_args.get("comparison_kind") or ("METRIC" if actual_args.get("compare_metric_id") else None)
+    for arg, exp_value in expected.comparison.items():
+        act_raw = kind if arg == "comparison_kind" else actual_args.get(arg)
+        defaults_apply = arg != "comparison_periods_ago" or str(kind).upper() in _PERIOD_COMPARISON_KINDS
+        if act_raw is None and kind is not None and defaults_apply:
+            act_raw = _COMPARISON_ARG_DEFAULTS.get(arg)
+        # Only the fixture side of an object id is trimmed: a padded id is a different id.
+        actual = act_raw if arg in _COMPARISON_ID_ARGS else _canonical_comparison_value(arg, act_raw)
+        if actual != exp_value:
+            return False
+    return True
+
+
+def _check_title(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+    """Exact match on the alert name when the fixture states one.
+
+    Only the fixture side is trimmed: a name saved with padding is not the name the user asked for.
+
+    An omitted ``title`` fails: ``create_metric_alert`` then saves an auto-generated name, even
+    when ``automation_id`` carries a slug of the requested one.
+    """
+    if expected.title is None:
+        return True
+    actual = actual_args.get("title")
+    return isinstance(actual, str) and actual == expected.title
+
+
+def _check_dashboard(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+    """Compare the bound dashboard id when the fixture states one; the actual id verbatim."""
+    if expected.dashboard_id is None:
+        return True
+    actual = actual_args.get("dashboard_id")
+    return isinstance(actual, str) and actual == expected.dashboard_id
+
+
 def _check_metric(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     if not expected.metric_id:
         return True
@@ -305,35 +419,14 @@ def _check_recipients(
 ) -> bool:
     if not expected.recipients:
         return True
-    act_recip_raw = actual_args.get("recipients", actual_args.get("external_recipients"))
-    if isinstance(act_recip_raw, str):
-        # external_recipients is JSON-encoded (e.g. '["email@example.com"]')
-        try:
-            parsed = json.loads(act_recip_raw)
-            act_recip = parsed if isinstance(parsed, list) else _parse_recipients(act_recip_raw)
-        except (json.JSONDecodeError, ValueError):
-            act_recip = _parse_recipients(act_recip_raw)
-    elif isinstance(act_recip_raw, list):
-        act_recip = act_recip_raw
-    else:
-        act_recip = []
-    act_internal_raw = actual_args.get("internal_recipients")
-    # internal_recipients is declared `anyOf: [array of string, string, null]` in the
-    # create_metric_alert tool schema -- a single id as a bare string is schema-legal,
-    # not a malformed call, so it needs the same string/list normalization already
-    # applied to recipients/external_recipients above.
-    if isinstance(act_internal_raw, str):
-        act_internal = [act_internal_raw]
-    elif isinstance(act_internal_raw, list):
-        act_internal = act_internal_raw
-    else:
-        act_internal = []
+    act_recip = _recipient_list(actual_args.get("recipients", actual_args.get("external_recipients")), split_bare=True)
+    act_internal = _recipient_list(actual_args.get("internal_recipients"), split_bare=False)
     wants_current_user, expected_emails = _split_current_user(expected.recipients)
     if wants_current_user:
         if current_user_id is None or current_user_id not in act_internal:
             return False
         other_internal = set(act_internal) - {current_user_id}
-        external = set(act_recip or [])
+        external = set(act_recip)
         if external - set(expected_emails):
             return False
         # An email not sent externally must arrive as its user's id (workspace-restricted channel).
@@ -345,7 +438,7 @@ def _check_recipients(
         # One lookup per email: a bulk lookup returns only the users that exist, hiding a missing one.
         ids_per_email = [_resolve_internal_recipient_ids(sdk, [email]) for email in unmatched]
         return all(ids_per_email) and other_internal == set().union(*ids_per_email)
-    if set(expected_emails) == set(act_recip or []):
+    if set(expected_emails) == set(act_recip):
         return True
     if sdk is not None and act_internal:
         internal_recipient_ids = _resolve_internal_recipient_ids(sdk, expected_emails)
@@ -489,6 +582,15 @@ def generate_simulated_alert_response(
 
     original_request = f'Your original request to the agent was: "{question}"\n' if question else ""
 
+    # Only a stated title is enforced: an unstated one is the agent's to choose, and telling the
+    # sim-user nothing keeps it from objecting to a generated name.
+    title_rule = (
+        f'8. The alert must be named exactly "{expected.title}". If a summary or proposal shows a '
+        "different name, do NOT confirm: state that exact name and ask the agent to use it.\n"
+        if expected.title
+        else ""
+    )
+
     system_prompt = (
         "You are a user requesting creation of an alert for a metric from an AI agent. "
         "Respond naturally but always steer toward the exact values you were given.\n"
@@ -509,7 +611,10 @@ def generate_simulated_alert_response(
         + recipients_rule
         + filters_rule
         + f"6. Proactively state how often you want to be alerted in your first reply: {trigger_request}. "
-        "   Repeat it if the agent proposes a different cadence.\n" + period_rule + "Reply concisely and directly."
+        "   Repeat it if the agent proposes a different cadence.\n"
+        + period_rule
+        + title_rule
+        + "Reply concisely and directly."
     )
 
     messages: list = [{"role": "system", "content": system_prompt}]
@@ -550,22 +655,28 @@ class AlertEvaluation:
     recipients_correct: bool
     attributes_correct: bool = True
     granularity_correct: bool = True
+    # ``None``: the item states no such expectation, so the check is neither scored nor gated.
+    comparison_correct: bool | None = None
+    title_correct: bool | None = None
+    dashboard_correct: bool | None = None
+
+    def checks(self) -> dict[str, bool]:
+        """Every check that ran, by name; strict_pass requires all of them.
+
+        The conditional checks are left out when unasserted, so an item that names no title does
+        not report a ``title_correct`` it was never scored on, and a refusal on it is not counted
+        as a title failure.
+        """
+        ran: dict[str, bool] = {name: getattr(self, name) for name in _ALERT_CHECKS}
+        for name in _CONDITIONAL_ALERT_CHECKS:
+            value = getattr(self, name)
+            if value is not None:
+                ran[name] = value
+        return ran
 
     @property
     def strict_pass(self) -> bool:
-        return all(
-            [
-                self.alert_created,
-                self.operator_correct,
-                self.threshold_correct,
-                self.trigger_correct,
-                self.filters_correct,
-                self.metric_correct,
-                self.recipients_correct,
-                self.attributes_correct,
-                self.granularity_correct,
-            ]
-        )
+        return all(self.checks().values())
 
 
 @dataclass
@@ -666,6 +777,68 @@ def _normalize_expected_attributes(expected: dict) -> list | None:
     raise ValueError(f"Attributes expectation must be a list or a display string, got {type(attributes).__name__}")
 
 
+def _normalize_expected_comparison(expected: dict) -> dict[str, str | int | float | None] | None:
+    """
+    * ``Comparison`` object       -> its stated properties, keyed by tool-argument name
+    * "None" / "no comparison"    -> ``{"comparison_kind": None}`` (stated: no comparison)
+    * absent, empty, or prose     -> ``None`` (unstated; comparison not asserted)
+
+    Each property of the object is asserted only when it carries a value, so an item can pin
+    the comparison kind without also pinning the date dataset the agent may pick.
+    """
+    comparison = _case_insensitive_get(expected, "comparison")
+    if comparison is None:
+        return None
+    if isinstance(comparison, str):
+        return {"comparison_kind": None} if any(kw in comparison.lower() for kw in _NO_COMPARISON_MARKERS) else None
+    if not isinstance(comparison, dict):
+        raise ValueError(f"Comparison expectation must be an object or a display string, got {comparison!r}")
+    stated: dict[str, str | int | float | None] = {}
+    for key, value in comparison.items():
+        name = str(key).strip().lower()
+        arg = _COMPARISON_ARGS.get(name) or (name if name in _COMPARISON_ARGS.values() else None)
+        if arg is None:
+            # Validated here so a misspelled property fails before the run spends an API call.
+            raise ValueError(f"Unrecognised comparison property {key!r}; expected one of {sorted(_COMPARISON_ARGS)}")
+        canonical = _canonical_comparison_value(arg, value)
+        if canonical is None and arg == "comparison_periods_ago" and str(value or "").strip():
+            raise ValueError(f"Comparison periods_ago must be a number, got {value!r}")
+        if canonical is not None:
+            stated[arg] = canonical
+    return stated or None
+
+
+def _normalize_expected_text(expected: dict, *keys: str) -> str | None:
+    """A stated string expectation, or ``None`` when absent or blank (unasserted)."""
+    value = _case_insensitive_get(expected, *keys)
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        raise ValueError(f"{keys[0]} expectation must be a string, got {value!r}")
+    return value.strip() or None
+
+
+def _normalize_expected_dashboard(expected: dict) -> str | None:
+    """
+    * ``dashboard_id``            -> that id
+    * ``Dashboard`` "Title (id)"  -> the id in the trailing parentheses, as ``Metric`` is read
+    * absent / empty              -> ``None`` (unstated; binding not asserted)
+
+    A ``Dashboard`` without a trailing ``(id)`` raises: a title alone can never equal the
+    ``dashboard_id`` the tool receives, so the item would fail on every run.
+    """
+    dashboard_id = _normalize_expected_text(expected, "dashboard_id")
+    if dashboard_id is not None:
+        return dashboard_id
+    display = _normalize_expected_text(expected, "dashboard")
+    if display is None:
+        return None
+    parsed = _parse_metric_id(display)
+    if parsed is None:
+        raise ValueError(f'Dashboard expectation must be "Title (id)", got {display!r}; use dashboard_id for a bare id')
+    return parsed
+
+
 def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
     """Parse expected_output dict into CatalogMetricAlert, accepting display-format or internal-format keys."""
     operator = _case_insensitive_get(expected, "operator") or "GREATER_THAN"
@@ -696,6 +869,9 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
     granularity = AnomalyDetectionGranularity.parse(
         _case_insensitive_get(expected, "granularity", "detection granularity")
     )
+    comparison = _normalize_expected_comparison(expected)
+    title = _normalize_expected_text(expected, "title")
+    dashboard_id = _normalize_expected_dashboard(expected)
 
     return CatalogMetricAlert(
         operator=operator,
@@ -708,6 +884,9 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
         filters=filters,
         attributes=attributes,
         granularity=granularity,
+        comparison=comparison,
+        title=title,
+        dashboard_id=dashboard_id,
     )
 
 
@@ -893,6 +1072,13 @@ def run_agentic_alert_skill(
                 and _check_recipients(expected, actual_args, sdk=sdk, current_user_id=current_user_id),
                 attributes_correct=tool_called and _check_attributes(expected, actual_args),
                 granularity_correct=tool_called and _check_granularity(expected, actual_args),
+                comparison_correct=None
+                if expected.comparison is None
+                else tool_called and _check_comparison(expected, actual_args),
+                title_correct=None if expected.title is None else tool_called and _check_title(expected, actual_args),
+                dashboard_correct=None
+                if expected.dashboard_id is None
+                else tool_called and _check_dashboard(expected, actual_args),
             )
             return AlertRunResult(
                 conversation_id=conv_id,
@@ -934,19 +1120,7 @@ def run_agentic_alert_skill(
     pass_power_k = all(r.eval.strict_pass for r in run_results)
     best = max(
         run_results,
-        key=lambda r: sum(
-            [
-                r.eval.alert_created,
-                r.eval.operator_correct,
-                r.eval.threshold_correct,
-                r.eval.trigger_correct,
-                r.eval.filters_correct,
-                r.eval.metric_correct,
-                r.eval.recipients_correct,
-                r.eval.attributes_correct,
-                r.eval.granularity_correct,
-            ]
-        ),
+        key=lambda r: sum(r.eval.checks().values()),
     )
     return AgenticAlertSummary(
         run_results=run_results,
@@ -966,17 +1140,8 @@ def _run_detail(run: AlertRunResult) -> dict:
     Extracted so a failing run is described by exactly the same keys as the winning run --
     for this kind that means the full per-field check breakdown and the arguments the run sent.
     """
-    ev = run.eval
     return {
-        "alert_created": ev.alert_created,
-        "operator_correct": ev.operator_correct,
-        "threshold_correct": ev.threshold_correct,
-        "trigger_correct": ev.trigger_correct,
-        "filters_correct": ev.filters_correct,
-        "metric_correct": ev.metric_correct,
-        "recipients_correct": ev.recipients_correct,
-        "attributes_correct": ev.attributes_correct,
-        "granularity_correct": ev.granularity_correct,
+        **run.eval.checks(),
         "actual_alert_arguments": run.actual_alert_arguments,
         # Why the loop stopped. alert_created=False alone cannot tell a refusal from a run
         # that hit max_iterations while still on track -- see LoopExit. Per-run rather than
@@ -1052,18 +1217,7 @@ def evaluate_agentic_alert_skill(
 
             for run_idx, run in enumerate(summary.run_results):
                 pt = ctx.trace(run.conversation_id)
-                ev = run.eval
-                strict_checks = {
-                    "alert_created": ev.alert_created,
-                    "operator_correct": ev.operator_correct,
-                    "threshold_correct": ev.threshold_correct,
-                    "trigger_correct": ev.trigger_correct,
-                    "filters_correct": ev.filters_correct,
-                    "metric_correct": ev.metric_correct,
-                    "recipients_correct": ev.recipients_correct,
-                    "attributes_correct": ev.attributes_correct,
-                    "granularity_correct": ev.granularity_correct,
-                }
+                strict_checks = run.eval.checks()
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
@@ -1112,13 +1266,8 @@ def evaluate_agentic_alert_skill(
         gate_note = gate_failure_note(gate, runs_passed, runs_effective)
         exc = AlertSkillAssertionError(
             f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. "
-            f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, "
-            f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, "
-            f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, "
-            f"recipients_correct={ev.recipients_correct}, "
-            f"attributes_correct={ev.attributes_correct}, "
-            f"granularity_correct={ev.granularity_correct}. "
-            f"Actual args: {best.actual_alert_arguments}"
+            + ", ".join(f"{name}={value}" for name, value in ev.checks().items())
+            + f". Actual args: {best.actual_alert_arguments}"
         )
         exc.reasoning_steps = best.reasoning_steps
         exc.conversation_id = best.conversation_id

gooddata_eval/core/agentic/document_skill.py

@@ -32,7 +32,7 @@ from gooddata_eval.core.agentic._trace_linker import (
 from gooddata_eval.core.agentic.dashboard_skill import _extract_tool_result, _skill_activated
 from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
-from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.config import DocumentFormat, ReasoningEffort, normalize_document_format
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run
 from gooddata_eval.core.langfuse.item_scope import item_scope
 from gooddata_eval.core.models import (
@@ -801,18 +801,24 @@ def run_agentic_document_skill(
     agent_id: str | None = None,
     judge: LLMJudge | None = None,
     user_context: dict | None = None,
+    document_format: DocumentFormat | None = None,
 ) -> AgenticDocumentSummary:
     """Run the document-skill agentic evaluation K times and return a summary.
 
     The narrative judge is built only for a fixture that states a ``narrative``, so an item
     without one needs neither the llm-judge extra nor ``OPENAI_API_KEY``.
 
+    ``document_format`` is sent on every turn of every run, see ``ChatClient``; None leaves
+    the format to the server's ``enablePublisherV2`` feature flag.
+
     Raises:
         ValueError: the fixture is unusable — see ``_validate_expectation`` and
-            ``_validate_user_context``.
+            ``_validate_user_context``, or ``document_format`` is not a format gen-ai
+            accepts. Raised before any chat request.
     """
     _validate_expectation(expected_output)
     _validate_user_context(user_context)
+    document_format = normalize_document_format(document_format)
     if judge is None and _has_narrative(expected_output):
         judge = LLMJudge(_NARRATIVE_EVALUATION_STEPS)
     run_results: list[DocumentRunResult] = []
@@ -823,6 +829,7 @@ def run_agentic_document_skill(
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
         user_context=user_context,
+        document_format=document_format,
     )
 
     try:
@@ -882,20 +889,32 @@ def evaluate_agentic_document_skill(
     gate: EvalGate = DEFAULT_GATE,
     judge: LLMJudge | None = None,
     user_context: dict | None = None,
+    document_format: DocumentFormat | None = None,
 ) -> AgenticEvalOutcome:
     """Run document-skill evaluation, log to Langfuse, and raise on failure.
 
     Returns the best run's outcome on success; on failure the same values are attached to the
     raised ``DocumentSkillAssertionError`` so callers can retrieve them either way.
 
+    ``document_format`` (``aac-v1``/``aac-v2``) asks gen-ai for that document content format
+    on every turn and is recorded as ``document_format`` in the dataset-run metadata; None
+    leaves it to the server's ``enablePublisherV2`` feature flag.
+
     Raises:
         DocumentSkillAssertionError: the gate did not pass.
-        ValueError: the fixture is unusable — see ``_validate_expectation`` and
-            ``_validate_user_context``. Raised before any
-            request, so it means a fixture to fix rather than a result to read.
+        ValueError: the fixture is unusable, see ``_validate_expectation`` and
+            ``_validate_user_context``. Raised before any chat request, so it means a
+            fixture to fix rather than a result to read. Also raised, equally before any
+            request, when the caller passes a ``document_format`` gen-ai does not accept.
         JudgeResponseError: the fixture states a narrative and the judge returned no readable
             verdict for any run -- an item without a result, not K failures.
     """
+    document_format = normalize_document_format(document_format)
+    if document_format is not None:
+        # Metadata only, not part of the run name: it tells runs apart only when their names
+        # already differ, so two runs that share a run_timestamp, dataset and model but not the
+        # format still collide on one dataset-run name.
+        run_metadata_extra = {**(run_metadata_extra or {}), "document_format": document_format}
     identity = RunIdentity(
         host,
         token,
@@ -921,6 +940,7 @@ def evaluate_agentic_document_skill(
             agent_id=agent_id,
             judge=judge,
             user_context=user_context,
+            document_format=document_format,
         )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/chat/sse_client.py

@@ -26,7 +26,12 @@ from urllib.parse import quote
 import httpx
 
 from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, float_env, int_env, retry_transient
-from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
+from gooddata_eval.core.config import (
+    DocumentFormat,
+    ReasoningEffort,
+    normalize_document_format,
+    normalize_reasoning_effort,
+)
 from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for
 from gooddata_eval.core.models import ChatResult, DatasetItem
 from gooddata_eval.core.timing import note_trace_latency
@@ -460,6 +465,7 @@ class ChatClient:
         reasoning_effort: ReasoningEffort | None = None,
         agent_id: str | None = None,
         user_context: dict[str, Any] | None = None,
+        document_format: DocumentFormat | None = None,
     ):
         """Create a chat client bound to one workspace.
 
@@ -476,6 +482,13 @@ class ChatClient:
         back to it: a caller that changes or clears the context per message builds the
         client without one. The server grounds answers in it only while
         the ``enableAiContextSetup`` feature flag is on for the organization.
+
+        ``document_format`` (``aac-v1``/``aac-v2``) is sent as ``options.documentFormat``
+        on every message and selects the document content format gen-ai drafts in for
+        that turn. gen-ai keeps a draft in the format of the turn that wrote it and a
+        turn in the other format cannot continue it, so follow-up and clarification
+        turns carry it too. When None the key is omitted and the server picks the format
+        from the ``enablePublisherV2`` feature flag. Any other value raises ``ValueError``.
         """
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
         self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()}
@@ -494,6 +507,7 @@ class ChatClient:
         self._client = httpx.Client(timeout=http_timeout)
         self._preserve_failed = preserve_failed
         self._reasoning_effort = normalize_reasoning_effort(reasoning_effort)
+        self._document_format = normalize_document_format(document_format)
         self._agent_id = agent_id
         self._user_context = user_context
         # Run index of each conversation this client has sent to, in first-send order: every
@@ -531,8 +545,13 @@ class ChatClient:
         join_headers, joined = self._join_item_trace(conversation_id)
         headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json", **join_headers}
         body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
+        options: dict[str, str] = {}
         if self._reasoning_effort is not None:
-            body["options"] = {"reasoningEffort": self._reasoning_effort}
+            options["reasoningEffort"] = self._reasoning_effort
+        if self._document_format is not None:
+            options["documentFormat"] = self._document_format
+        if options:
+            body["options"] = options
         # A per-call context overrides the client's. Only sent when there is one: gen-ai
         # accepts an explicit null, so assigning unconditionally would quietly change every
         # request that has no attachment.

gooddata_eval/core/config.py

@@ -33,6 +33,9 @@ def judge_model() -> str:
 ReasoningEffort = Literal["LOW", "MEDIUM", "HIGH"]
 """Effort values the AI chat endpoint accepts, uppercase as the server enum requires."""
 
+DocumentFormat = Literal["aac-v1", "aac-v2"]
+"""Document content formats the AI chat endpoint accepts as ``options.documentFormat``."""
+
 EvalGate = Literal["any", "power"]
 """`any` = pass@K, `power` = pass^K. Behaviour is in core/agentic/_gate.py; the type lives here
 because putting it there would make config import the agentic package, whose __init__ imports
@@ -73,6 +76,22 @@ def normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None:
     return cast("ReasoningEffort", candidate)
 
 
+def normalize_document_format(value: str | None) -> DocumentFormat | None:
+    """Canonical document format, or None when unset.
+
+    gen-ai rejects any value outside `DocumentFormat`, so an unknown value raises here,
+    before a run sends its first request.
+    """
+    if value is None:
+        return None
+    candidate = value.strip().lower()
+    if not candidate:
+        return None
+    if candidate not in get_args(DocumentFormat):
+        raise ValueError(f"Invalid document format {value!r}; expected one of {', '.join(get_args(DocumentFormat))}.")
+    return cast("DocumentFormat", candidate)
+
+
 @dataclass
 class RunConfig:
     host: str

{gooddata_eval-1.78.1.dev2.dist-info → gooddata_eval-1.78.1.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.78.1.dev2
+Version: 1.78.1.dev4
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.78.1.dev2
+Requires-Dist: gooddata-sdk~=1.78.1.dev4
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15

{gooddata_eval-1.78.1.dev2.dist-info → gooddata_eval-1.78.1.dev4.dist-info}/RECORD

@@ -6,7 +6,7 @@ gooddata_eval/cli/main.py,sha256=ZZv9pViwV4h21LPMrBOUCH99-oyM9dd2AFpGj9NWEJI,344
 gooddata_eval/core/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/_output.py,sha256=sFbNtKg1lx431jpb15jISLlOXGTZ-r_KodfaTVyka6I,580
 gooddata_eval/core/_retry.py,sha256=86ZPWu6ahCYC2GLvcbwjHTtj2RrGPOuxIFlCbtmNpvw,2990
-gooddata_eval/core/config.py,sha256=HegNbWb4csXXGaOZiFkplhsr8FmVK7OJL_lDKne1Sdg,4080
+gooddata_eval/core/config.py,sha256=hla5t294uHC5Q9bzrhPjsdkwEcetLsEH-GqGAhs94b4,4823
 gooddata_eval/core/connection.py,sha256=mh-Do5-Ti0-DXXikfCIH10eTTUbKuZUAS_sLoPNDJ5I,1229
 gooddata_eval/core/granularity.py,sha256=CbniwgxtVj0oyf41UjeeZbBzOpeisSBlAfdkL79e6O0,3932
 gooddata_eval/core/interaction.py,sha256=PUaJ1zdMwXBKjoNXNFXVgt_b8_i0LWdbMZndEQ3F6UM,2093
@@ -17,17 +17,17 @@ gooddata_eval/core/scoring.py,sha256=buY7ZoO08Dt59m2PPhFeNOyix5yYlf4mVxFyOn-aibk
 gooddata_eval/core/timing.py,sha256=1grXpr48OH4GcAmeobhjwnRWqoMJwcjj-LLpHdS_raY,7789
 gooddata_eval/core/workspace.py,sha256=S8exiy5gpDtcRtT8-i2fIRALF34Pbo0J46xm8X_ldus,11846
 gooddata_eval/core/agentic/__init__.py,sha256=i2g22ousTYXi5FZeN5X0WVpa67RQXThwli8KcYfIYxM,4432
-gooddata_eval/core/agentic/_catalog.py,sha256=bzDGtiXVHdtJSSMCzV8qYp4p_BxmO7f7GifGNTPfvSA,3687
+gooddata_eval/core/agentic/_catalog.py,sha256=My4SP5DteSJlp2WOQ66iemJkX3ANmdqrsgANDEm6u10,4194
 gooddata_eval/core/agentic/_conversation_context.py,sha256=43ryGOQ9kTLLrzRxBphUn_87290vTZQ6oqWLnzjKYbA,16270
 gooddata_eval/core/agentic/_failed_runs.py,sha256=iXN80ORMrIKEhV1QKGbyEZPP2w__mLPSrkm3hh-dTwM,3048
 gooddata_eval/core/agentic/_gate.py,sha256=irI4eWOCfmsbKd6l-HdpoAlnWkq2wjo4b5crGU_Er1o,2631
 gooddata_eval/core/agentic/_langfuse.py,sha256=NWhPgpd2NkYmAPBiT8VCOd_cddhhsmDGmcWmsvyGLGs,29729
 gooddata_eval/core/agentic/_trace_linker.py,sha256=mSY_7UDZU97r5qcaDxMoyPQ46944XzbzFbMRd74G87k,15760
-gooddata_eval/core/agentic/alert_skill.py,sha256=5bsyWnHJVxDFk-Tr1e31F3P8vHB4HXJaM5TtrWue35Y,52537
+gooddata_eval/core/agentic/alert_skill.py,sha256=Me7YrAAGWG6iqzJR18JdhVsri4LG4J1itIUdzgboZU0,60105
 gooddata_eval/core/agentic/anomaly_detection.py,sha256=H_rKx6fCX3S11HzGAQGhy092YMmGj4foFBmrx6U-LHE,27701
 gooddata_eval/core/agentic/conversation.py,sha256=_OLlST1twnSPnOQGfuuFCFKu92uwLVh2YBBA_nOW-5w,60004
 gooddata_eval/core/agentic/dashboard_skill.py,sha256=PnFq1JjaMpAjQkzgklRkEFFKEs4WWhORikDVw5qjbaE,84630
-gooddata_eval/core/agentic/document_skill.py,sha256=Z9jzWhOjaPmQze1IeDrc08pg0XB31_NTWBRQkxXf178,43859
+gooddata_eval/core/agentic/document_skill.py,sha256=Uukqc6voa3lXbcSVgfzQyPH_kEFu3jrbrN9EDIMIuYA,45231
 gooddata_eval/core/agentic/general_question.py,sha256=pOVYlJF9eKZwTqdPVsrH4X1OC-tXI7CMCFDdPNdBIMw,17758
 gooddata_eval/core/agentic/guardrail.py,sha256=1Yz5QVVWo5GCOA8OKSy5efeysaaXB3bXcr_BT-Qu8tY,16398
 gooddata_eval/core/agentic/kda_skill.py,sha256=2mav9itclavID28U4fORSR9p-iVTrddMyTHLKh8HxdE,26154
@@ -38,7 +38,7 @@ gooddata_eval/core/agentic/visualization.py,sha256=o4RVS-nRCVbVXjbE_jHuGzgW8x3hR
 gooddata_eval/core/agentic/what_if.py,sha256=v5u9Yl45QGpVD2IHvPhVW_XOixI10IK4N-5_zP9Yymw,26365
 gooddata_eval/core/chat/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/chat/render.py,sha256=CqsKN8__DlaiJj_bmcPZ1poc5LfUqtz7EzKRDbks4gU,2082
-gooddata_eval/core/chat/sse_client.py,sha256=UGmghNUQHGM5NMBtAAJj6Z9jQpgs0ROfqH9egw3Tgvo,31006
+gooddata_eval/core/chat/sse_client.py,sha256=yGUbWg7gr7ycVfEpvpmGUCIQ38nW0jiMmuJB9_va4UY,31921
 gooddata_eval/core/dataset/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/dataset/from_insights.py,sha256=2eYh_84070t2BMeW2XPLAwzrJs7vKbFYMeCppvkDiEo,40880
 gooddata_eval/core/dataset/langfuse_source.py,sha256=II-6fenEPhdyfckrL0iVdJpavjv67wUvy7OIuvHntrg,8697
@@ -77,8 +77,8 @@ gooddata_eval/core/summary/http_client.py,sha256=Umae42fiiJQ0e4hv63NRROHYvcFAGgX
 gooddata_eval/core/takeaway/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/takeaway/evaluate.py,sha256=ei7DLSlG_Ove_R551Ofdw8WISf178XknU58jBWxB_is,11585
 gooddata_eval/core/takeaway/http_client.py,sha256=yc3FNPArjfcSdAE7naRf7g_aW7BbJhwsfSEvAFd67Ik,4141
-gooddata_eval-1.78.1.dev2.dist-info/METADATA,sha256=n-O7qyx_gp7werx53dJUr2y_GyY3dooBZlUDlrwhLnk,60351
-gooddata_eval-1.78.1.dev2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-gooddata_eval-1.78.1.dev2.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
-gooddata_eval-1.78.1.dev2.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
-gooddata_eval-1.78.1.dev2.dist-info/RECORD,,
+gooddata_eval-1.78.1.dev4.dist-info/METADATA,sha256=uQZy3GjLJ3V1W9B9RR9kX6fLzD5-Var8lEQApCzH3iA,60351
+gooddata_eval-1.78.1.dev4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+gooddata_eval-1.78.1.dev4.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
+gooddata_eval-1.78.1.dev4.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
+gooddata_eval-1.78.1.dev4.dist-info/RECORD,,