gooddata-eval 1.76.1.dev1__py3-none-any.whl → 1.76.1.dev3__py3-none-any.whl

gooddata_eval/cli/agentic_runner.py

@@ -24,6 +24,7 @@ from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
 from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.evaluators.visualization import requires_execution_of
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
@@ -179,9 +180,11 @@ def _dispatch_agentic(
             workspace_id=workspace_id,
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
+            requires_execution=requires_execution_of(eo),
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_metric_skill":
@@ -194,6 +197,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_dashboard_skill":
@@ -206,6 +210,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_report_skill":
@@ -218,6 +223,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_alert_skill":
@@ -230,6 +236,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_search":
@@ -245,6 +252,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_general_question":
@@ -270,6 +278,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_kda_skill":
@@ -282,6 +291,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_what_if":
@@ -293,6 +303,7 @@ def _dispatch_agentic(
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_anomaly_detection":
@@ -305,6 +316,7 @@ def _dispatch_agentic(
             k=k,
             gate=gate,
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     elif kind == "agentic_conversation":
@@ -315,6 +327,7 @@ def _dispatch_agentic(
             workspace_id=workspace_id,
             fixture=ConversationFixture.model_validate(fixture_data),
             agent_id=agent_id,
+            user_context=item.user_context,
             **lf_kw,
         )
     else:

gooddata_eval/core/agentic/alert_skill.py

@@ -659,12 +659,18 @@ def run_agentic_alert_skill(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticAlertSummary:
     """Run the alert-skill agentic evaluation K times and return a summary."""
     expected = _normalize_expected_output(expected_output)
     run_results: list[AlertRunResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
     sdk = GoodDataSdk.create(host, token)
 
@@ -858,6 +864,7 @@ def evaluate_agentic_alert_skill(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.
@@ -881,6 +888,7 @@ def evaluate_agentic_alert_skill(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/agentic/anomaly_detection.py

@@ -344,6 +344,7 @@ def run_agentic_anomaly_detection(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticAnomalySummary:
     """Run the anomaly-detection agentic evaluation K times and return a summary.
 
@@ -356,7 +357,12 @@ def run_agentic_anomaly_detection(
         raise ValueError(f"k must be >= 1, got {k}")
     run_results: list[AnomalyRunResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
 
     def _run_once(conv_id: str) -> AnomalyRunResult:
@@ -520,6 +526,7 @@ def evaluate_agentic_anomaly_detection(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
 ) -> AgenticEvalOutcome:
     """Run anomaly-detection evaluation, log to Langfuse, and raise on failure."""
     langfuse, window_start = open_trace_window(langfuse)
@@ -534,6 +541,7 @@ def evaluate_agentic_anomaly_detection(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/agentic/conversation.py

@@ -24,7 +24,7 @@ from datetime import date
 from typing import ClassVar, Literal
 
 from gooddata_sdk import GoodDataSdk
-from pydantic import BaseModel, Field, ValidationError
+from pydantic import BaseModel, Field, ValidationError, model_validator
 
 from gooddata_eval.core.agentic._conversation_context import (
     CONFIRMATION_REPLY,
@@ -100,6 +100,12 @@ class TurnDefinition(BaseModel):
     over. ``set_answers`` are what the user replies, in order, when the assistant asks a
     legitimate question on this turn -- the only knowledge the context-mode simulated user
     has beyond the conversation itself.
+
+    ``user_context`` is the ``userContext`` (dashboard/widget scope) the user has attached
+    from this turn on, as in the UI: it sticks to later turns until another turn sets it,
+    and setting it to null clears it. A turn that leaves it out keeps the current one. Its
+    clarification replies carry it too, since gen-ai treats a message without a context as
+    cleared.
     """
 
     turn_id: str
@@ -112,6 +118,7 @@ class TurnDefinition(BaseModel):
     expected_tool_args: dict | None = None
     depends_on: list[str] = Field(default_factory=list)
     set_answers: list[str] = Field(default_factory=list)
+    user_context: dict | None = None
 
 
 class ConversationFixture(BaseModel):
@@ -122,6 +129,17 @@ class ConversationFixture(BaseModel):
     expected_skills: list[str]
     turns: list[TurnDefinition]
 
+    @model_validator(mode="before")
+    @classmethod
+    def _reject_fixture_level_user_context(cls, data: object) -> object:
+        # Unknown keys are ignored, so a context placed here would be dropped without a trace.
+        if isinstance(data, dict) and "user_context" in data:
+            raise ValueError(
+                "user_context is not read from the conversation fixture; put it in the item "
+                "metadata or input (the context of the first turn), or on a turn"
+            )
+        return data
+
 
 ReplyRecordKind = Literal["confirmation", "question", "no_action", "turn_incomplete"]
 
@@ -696,6 +714,7 @@ def run_agentic_conversation(
     mode: ConversationMode | None = None,
     clarification_judge: ClarificationJudge | None = None,
     fresh_conversation_per_turn: bool = False,
+    user_context: dict | None = None,
 ) -> ConversationResult:
     """Run a multi-turn, multi-skill conversation evaluation (no K-runs).
 
@@ -706,6 +725,9 @@ def run_agentic_conversation(
     ``fresh_conversation_per_turn`` sends every turn into a new conversation instead, so the
     agent sees no history. It is the no-memory baseline a context score is calibrated
     against: context-dependent turns are expected to fail there.
+
+    ``user_context`` is the context attached when the conversation starts; a turn's own
+    ``user_context`` replaces it from that turn on.
     """
     resolved_mode = resolve_conversation_mode(mode)
     if fresh_conversation_per_turn and initial_conversation_id is not None:
@@ -747,6 +769,7 @@ def run_agentic_conversation(
     turn_offset = 0.0
     tool_index_offset = 0
     reasoning_index_offset = 0
+    current_context = user_context
 
     try:
         if initial_conversation_id is not None:
@@ -761,6 +784,10 @@ def run_agentic_conversation(
                 conversation_id = client.create_conversation()
                 transcript = []
                 active_skills = set()
+            # Before the skip below: a turn whose expectation cannot resolve still changed what
+            # the user has attached. Only a turn that names the field changes it; null clears it.
+            if "user_context" in turn.model_fields_set:
+                current_context = turn.user_context
             try:
                 resolved_expected = _resolve_refs(turn.expected_output, turn_outputs)
                 resolved_alternatives = [
@@ -813,7 +840,7 @@ def run_agentic_conversation(
                 transcript.append(TranscriptEntry("user", current_message))
                 incomplete = False
                 try:
-                    chat_result = client.send_message(conversation_id, current_message)
+                    chat_result = client.send_message(conversation_id, current_message, user_context=current_context)
                 except TurnIncompleteError as exc:
                     chat_result = exc.partial_result or ChatResult()
                     incomplete = True
@@ -1054,6 +1081,7 @@ def evaluate_agentic_conversation(
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
     mode: ConversationMode | None = None,
     clarification_judge: ClarificationJudge | None = None,
+    user_context: dict | None = None,
 ) -> AgenticEvalOutcome:
     """Run conversation evaluation, log to Langfuse, and raise on failure.
 
@@ -1078,6 +1106,7 @@ def evaluate_agentic_conversation(
         agent_id=agent_id,
         mode=resolved_mode,
         clarification_judge=clarification_judge,
+        user_context=user_context,
     )
     passed = result.context_success if resolved_mode == "context" else result.conversation_success
 

gooddata_eval/core/agentic/dashboard_skill.py

@@ -955,6 +955,7 @@ def run_agentic_dashboard_skill(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticDashboardSummary:
     """Run the dashboard-skill agentic evaluation K times and return a summary.
 
@@ -964,7 +965,12 @@ def run_agentic_dashboard_skill(
     _validate_expectation(expected_output)
     run_results: list[DashboardRunResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
 
     try:
@@ -1021,6 +1027,7 @@ def evaluate_agentic_dashboard_skill(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run dashboard-skill evaluation, log to Langfuse, and raise on failure.
@@ -1045,6 +1052,7 @@ def evaluate_agentic_dashboard_skill(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/agentic/general_question.py

@@ -116,11 +116,10 @@ def _run_single_general_question(
     conversation_id: str,
     question: str,
     expected_output: str,
-    user_context: dict | None = None,
 ) -> GeneralQuestionResult:
     item_started = time.monotonic()
     agent_started = time.monotonic()
-    chat_result = client.send_message(conversation_id, question, user_context=user_context)
+    chat_result = client.send_message(conversation_id, question)
     actual_output = render_answer_text(chat_result)
     agent_elapsed = time.monotonic() - agent_started
     log_timer(
@@ -169,16 +168,19 @@ def run_agentic_general_question(
     """Run the general-question agentic evaluation K times and return a summary."""
     run_results: list[GeneralQuestionResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
     judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS)
 
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
-            run_results.append(
-                _run_single_general_question(client, judge, conv_id_0, question, expected_output, user_context)
-            )
+            run_results.append(_run_single_general_question(client, judge, conv_id_0, question, expected_output))
         finally:
             if initial_conversation_id is None:
                 client.delete_conversation(conv_id_0)
@@ -186,9 +188,7 @@ def run_agentic_general_question(
         for _ in range(1, k):
             conv_id = client.create_conversation()
             try:
-                run_results.append(
-                    _run_single_general_question(client, judge, conv_id, question, expected_output, user_context)
-                )
+                run_results.append(_run_single_general_question(client, judge, conv_id, question, expected_output))
             finally:
                 client.delete_conversation(conv_id)
     finally:

gooddata_eval/core/agentic/guardrail.py

@@ -144,11 +144,17 @@ def run_agentic_guardrail(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticGuardrailSummary:
     """Run the guardrail agentic evaluation K times and return a summary."""
     run_results: list[GuardrailResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
     judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS)
 
@@ -206,6 +212,7 @@ def evaluate_agentic_guardrail(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run guardrail evaluation, log to Langfuse, and raise GuardrailAssertionError on failure.
@@ -226,6 +233,7 @@ def evaluate_agentic_guardrail(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/agentic/kda_skill.py

@@ -239,6 +239,7 @@ def run_agentic_kda_skill(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticKdaSummary:
     """Run the KDA-skill agentic evaluation K times and return a summary.
 
@@ -256,7 +257,12 @@ def run_agentic_kda_skill(
         raise ValueError(f"k must be >= 1, got {k}")
     run_results: list[KdaRunResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
 
     def _run_once(conv_id: str) -> KdaRunResult:
@@ -424,6 +430,7 @@ def evaluate_agentic_kda_skill(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run KDA-skill evaluation, log to Langfuse, and raise KdaSkillAssertionError on failure.
@@ -444,6 +451,7 @@ def evaluate_agentic_kda_skill(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/agentic/metric_skill.py

@@ -9,6 +9,7 @@ import time
 from dataclasses import dataclass, field
 from typing import Any
 
+from gooddata_api_client.exceptions import NotFoundException
 from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.agentic._gate import (
@@ -50,6 +51,9 @@ except ImportError:
 
 _DEFAULT_K = 1
 _DEFAULT_MAX_ITERATIONS = 7
+# Seconds to wait before each check that a deleted metric stayed deleted, so the checks fall
+# 3, 6 and 12 seconds after the delete; see _delete_metric.
+_DELETE_RECHECK_DELAYS_S: tuple[float, ...] = (3.0, 3.0, 6.0)
 
 
 def _best_maql_match(actual_maql: str, expected_outputs: list[dict]) -> tuple[bool, str]:
@@ -238,11 +242,53 @@ def _delete_metric(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> None:
     MAQL) and the assertion fails. Deleting the created metric on the way out keeps
     the workspace clean for the next run. Best-effort: failures are logged, not raised.
     Mirrors ``alert_skill._delete_alert``.
+
+    A delete does not stay done on its own. ``create_metric`` reads the whole analytics
+    model and writes it back, so a concurrent test whose read preceded this delete
+    restores the metric when its write lands. The metric is therefore checked again after
+    each of ``_DELETE_RECHECK_DELAYS_S`` and deleted again while it is still there.
     """
+    if not _try_delete_metric(sdk, workspace_id, metric_id):
+        return
+    for delay in _DELETE_RECHECK_DELAYS_S:
+        time.sleep(delay)
+        if not _metric_exists(sdk, workspace_id, metric_id):
+            return
+        print(f"[CLEANUP] Metric {metric_id} is back or could not be checked; deleting again")
+        if not _try_delete_metric(sdk, workspace_id, metric_id):
+            return
+    if _DELETE_RECHECK_DELAYS_S:
+        # Every check found it back, so the last delete is unverified: say so rather than
+        # report a clean workspace. The post-run reset is what removes it for good.
+        print(
+            f"[CLEANUP] Metric {metric_id} was back or unchecked at all {len(_DELETE_RECHECK_DELAYS_S)} checks; "
+            "the last delete is not verified and the metric may remain"
+        )
+
+
+def _try_delete_metric(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> bool:
+    """Delete the metric; True when it is gone afterwards, a 404 included."""
     try:
         sdk._client.entities_api.delete_entity_metrics(workspace_id, metric_id)
+    except NotFoundException:
+        return True
     except Exception as exc:
         print(f"[CLEANUP] Failed to delete metric {metric_id}: {exc}")
+        return False
+    return True
+
+
+def _metric_exists(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> bool:
+    """Whether the metric may still be in the workspace. Only a 404 reads as absent; any other
+    lookup failure reads as present, so the caller deletes again rather than leave a restored
+    metric behind. The recheck delays bound how often that happens."""
+    try:
+        sdk._client.entities_api.get_entity_metrics(workspace_id, metric_id)
+    except NotFoundException:
+        return False
+    except Exception as exc:
+        print(f"[CLEANUP] Could not check metric {metric_id}: {exc}")
+    return True
 
 
 def _execute_single_metric_run(
@@ -396,6 +442,7 @@ def run_agentic_metric_skill(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticMetricSummary:
     """Run the metric-skill agentic evaluation K times and return a summary.
 
@@ -405,7 +452,12 @@ def run_agentic_metric_skill(
     expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output]
     run_results: list[MetricRunResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
     sdk = GoodDataSdk.create(host, token)
 
@@ -467,6 +519,7 @@ def evaluate_agentic_metric_skill(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.
@@ -490,6 +543,7 @@ def evaluate_agentic_metric_skill(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/agentic/report_skill.py

@@ -625,6 +625,7 @@ def run_agentic_report_skill(
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
     judge: LLMJudge | None = None,
+    user_context: dict | None = None,
 ) -> AgenticReportSummary:
     """Run the report-skill agentic evaluation K times and return a summary.
 
@@ -639,7 +640,12 @@ def run_agentic_report_skill(
         judge = LLMJudge(_NARRATIVE_EVALUATION_STEPS)
     run_results: list[ReportRunResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
 
     try:
@@ -698,6 +704,7 @@ def evaluate_agentic_report_skill(
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
     gate: EvalGate = DEFAULT_GATE,
     judge: LLMJudge | None = None,
+    user_context: dict | None = None,
 ) -> AgenticEvalOutcome:
     """Run report-skill evaluation, log to Langfuse, and raise on failure.
 
@@ -724,6 +731,7 @@ def evaluate_agentic_report_skill(
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
         judge=judge,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/agentic/search_tool.py

@@ -97,12 +97,18 @@ def run_agentic_search_tool(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticSearchSummary:
     """Run the search-tool agentic evaluation K times (single-turn each)."""
     run_results: list[SearchResult] = []
 
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
@@ -185,6 +191,7 @@ def evaluate_agentic_search_tool(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure.
@@ -204,6 +211,7 @@ def evaluate_agentic_search_tool(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/agentic/visualization.py

@@ -35,6 +35,7 @@ from gooddata_eval.core.evaluators.visualization import (
     _check_visualization_skill_activated,
     _evaluate_against_candidates,
     evaluation_result_detail,
+    with_execution,
 )
 from gooddata_eval.core.models import (
     AgenticAssertionError,
@@ -189,6 +190,7 @@ def _execute_single_run(
     question: str,
     expected_outputs: list[CreatedVisualization],
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
+    requires_execution: bool = False,
 ) -> RunResult:
     """Drive one full multi-turn conversation and evaluate the result."""
     total_turns = 0
@@ -214,6 +216,9 @@ def _execute_single_run(
         # K-run already completed along with any exit_reason.
         print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}")
         current_result = getattr(exc, "partial_result", None) or ChatResult()
+        # The loop that collects events does not run after CHAT_ERROR, so a partial result's
+        # chart executions are added here.
+        all_tool_call_events.extend(current_result.tool_call_events)
         exit_reason = LoopExit.CHAT_ERROR
     # Counted here, not at the top of the loop: this request is sent unconditionally, so
     # with max_iterations=0 the loop body never runs and total_turns would report 0 turns
@@ -265,6 +270,9 @@ def _execute_single_run(
             partial = getattr(exc, "partial_result", None)
             if partial is not None:
                 current_result = partial
+                # The loop adds a turn's events at its next pass, which a break skips. The
+                # partial result is scored, so its chart executions count too.
+                all_tool_call_events.extend(partial.tool_call_events)
             exit_reason = LoopExit.CHAT_ERROR
             break
 
@@ -274,6 +282,7 @@ def _execute_single_run(
         actual_output = current_result.created_visualizations.objects[0]
 
     eval_result, best_expected = _evaluate_against_candidates(expected_outputs, actual_output, skill_activated)
+    eval_result = with_execution(eval_result, all_tool_call_events, current_result.text_response, requires_execution)
 
     return RunResult(
         conversation_id=conversation_id,
@@ -301,6 +310,8 @@ def run_agentic_visualization(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
+    requires_execution: bool = False,
 ) -> AgenticRunSummary:
     """Run K independent conversations and return evaluation results.
 
@@ -308,16 +319,25 @@ def run_agentic_visualization(
     (e.g. one created by a Tavern YAML POST). Subsequent runs always create
     fresh conversations. Caller-supplied conversations are not deleted; all
     conversations created by this function are deleted on completion.
+
+    ``requires_execution`` fails a run whose agent built the chart but never ran it.
     """
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
     run_results: list[RunResult] = []
 
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
-            run_results.append(_execute_single_run(client, conv_id_0, question, expected_outputs, max_iterations))
+            run_results.append(
+                _execute_single_run(client, conv_id_0, question, expected_outputs, max_iterations, requires_execution)
+            )
         finally:
             if initial_conversation_id is None:
                 client.delete_conversation(conv_id_0)
@@ -325,7 +345,9 @@ def run_agentic_visualization(
         for _ in range(1, k):
             conv_id = client.create_conversation()
             try:
-                run_results.append(_execute_single_run(client, conv_id, question, expected_outputs, max_iterations))
+                run_results.append(
+                    _execute_single_run(client, conv_id, question, expected_outputs, max_iterations, requires_execution)
+                )
             finally:
                 client.delete_conversation(conv_id)
     finally:
@@ -378,6 +400,8 @@ def evaluate_agentic_visualization(
     record_output_path: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
+    requires_execution: bool = False,
     gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure.
@@ -400,6 +424,8 @@ def evaluate_agentic_visualization(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
+        requires_execution=requires_execution,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -420,6 +446,8 @@ def evaluate_agentic_visualization(
                     "assertion-vis-filters": ev.filters_correct,
                     "assertion-vis-type": ev.viz_type_hard,
                 }
+                if ev.requires_execution:
+                    strict_checks["assertion-vis-executed"] = ev.executed
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
                     ctx.score(tid, name="assertion-cross-ref-valid", value=ev.cross_ref_valid, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-metric", value=ev.metrics_correct, data_type="BOOLEAN")
@@ -427,6 +455,10 @@ def evaluate_agentic_visualization(
                     ctx.score(tid, name="assertion-vis-filters", value=ev.filters_correct, data_type="BOOLEAN")
                     ctx.score(tid, name="assertion-vis-type", value=ev.viz_type_hard, data_type="BOOLEAN")
                     ctx.score(tid, name="skill_selection", value=ev.skill_activated, data_type="BOOLEAN")
+                    # Scored on every item, gating or not, so the run rate is visible per model.
+                    ctx.score(tid, name="assertion-vis-executed", value=ev.executed, data_type="BOOLEAN")
+                    if ev.stated_value_matches is not None:
+                        ctx.score(tid, name="stated-value-matches", value=ev.stated_value_matches, data_type="BOOLEAN")
                     # Superseded by log_gate_scores' K-stable names, kept until the readers
                     # migrate: gdc-nas combo_report.py matches on the literal "pass_at_2".
                     ctx.score(tid, name=f"pass_at_{K}", value=summary.pass_at_k, data_type="BOOLEAN")
@@ -524,6 +556,8 @@ def evaluate_agentic_visualization(
             f"    attribute : {ev.filter_attribute_score}\n"
             f"{_filter_diff('attribute', ev)}"
             f"  Viz Type Hard         : {ev.viz_type_hard}\n"
+            f"  Executed              : {ev.executed}{' (required)' if ev.requires_execution else ''}\n"
+            f"  Stated Value Matches  : {ev.stated_value_matches} (reported only)\n"
             "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
         )
         exc.reasoning_steps = best.reasoning_steps

gooddata_eval/core/agentic/what_if.py

@@ -293,6 +293,7 @@ def run_agentic_what_if(
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
+    user_context: dict | None = None,
 ) -> AgenticWhatIfSummary:
     """Run the what-if agentic evaluation K times and return a summary.
 
@@ -305,7 +306,12 @@ def run_agentic_what_if(
         raise ValueError(f"k must be >= 1, got {k}")
     run_results: list[WhatIfRunResult] = []
     client = ChatClient(
-        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
+        user_context=user_context,
     )
 
     def _run_once(conv_id: str) -> WhatIfRunResult:
@@ -472,6 +478,7 @@ def evaluate_agentic_what_if(
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    user_context: dict | None = None,
 ) -> AgenticEvalOutcome:
     """Run what-if evaluation, log to Langfuse, and raise WhatIfAssertionError on failure."""
     langfuse, window_start = open_trace_window(langfuse)
@@ -486,6 +493,7 @@ def evaluate_agentic_what_if(
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
         agent_id=agent_id,
+        user_context=user_context,
     )
 
     if langfuse is not None and dataset_item_id:

gooddata_eval/core/chat/sse_client.py

@@ -13,12 +13,15 @@ module is the single place to swap — the runner only depends on the ChatBacken
 protocol, not on this class.
 """
 
+import functools
 import json
 import logging
 import os
+import re
 import time
 from dataclasses import dataclass, field
 from typing import Any, Callable, Iterable, TypeVar
+from urllib.parse import quote
 
 import httpx
 
@@ -135,6 +138,45 @@ _TURN_TIMEOUT_S = _float_env("GOODDATA_EVAL_CHAT_TURN_TIMEOUT_S", 0.0)
 # 4-turn conversation run to 4x the budget, which is not what a user would sit through.
 _ITEM_TIMEOUT_S = _float_env("GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S", 0.0)
 
+# Comma-separated `key=value` labels, e.g. `model_version=gpt-5.2,github_run_id=42`, sent as
+# W3C baggage on every chat request. gen-ai's Langfuse span processor copies `langfuse_*`
+# baggage onto each span it starts, so every observation of the conversation -- the cost-bearing
+# generations, retries and the title trace included -- carries them from creation. Langfuse v4
+# cannot add them to an observation afterwards. A value cannot contain a comma, and a key must be
+# a plain token (letters, digits, `_`, `-`); any other key is skipped with a warning, because httpx
+# refuses a non-ASCII header and gen-ai's OTel propagator drops or rewrites the rest.
+_TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS"
+_TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+")
+
+
+@functools.cache
+def _warn_skipped_trace_label(key: str) -> None:
+    """Warn once per key per process: every ChatClient re-reads the labels."""
+    _log.warning("%s: skipping label with key %r, keys must match [A-Za-z0-9_-]+", _TRACE_LABELS_ENV, key)
+
+
+def _trace_baggage() -> dict[str, str]:
+    """The `baggage` header for the labels in ``GOODDATA_EVAL_TRACE_LABELS``, or none.
+
+    Each label becomes trace metadata, and ``model_version`` -- the combo the report groups
+    on -- also becomes the observation ``version``, the dimension Langfuse groups cost by.
+    Values are percent-encoded: a space or comma would otherwise end the baggage entry.
+    """
+    labels: dict[str, str] = {}
+    for pair in os.getenv(_TRACE_LABELS_ENV, "").split(","):
+        key, sep, value = pair.partition("=")
+        key = key.strip()
+        if not sep or not key:
+            continue
+        if _TRACE_LABEL_KEY.fullmatch(key):
+            labels[key] = value.strip()
+        else:
+            _warn_skipped_trace_label(key)
+    entries = [f"langfuse_metadata_{k}={quote(v, safe='')}" for k, v in labels.items()]
+    if labels.get("model_version"):
+        entries.insert(0, f"langfuse_version={quote(labels['model_version'], safe='')}")
+    return {"baggage": ",".join(entries)} if entries else {}
+
 
 def set_default_turn_timeout(seconds: float | None) -> None:
     """Set the per-turn budget every ChatClient built afterwards inherits.
@@ -308,11 +350,15 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
         # create_adhoc_visualization but the call failed (e.g. data source not
         # accessible). The last attempt is the agent's best answer.
         #
-        # These are raw tool-call arguments, so they carry no `id` -- nothing was
-        # ever persisted. CreatedVisualization requires one, so synthesize a
-        # sentinel rather than letting the whole ChatResult fail to validate:
-        # dropping the turn entirely would score a stalled data source as a
-        # content failure, which is exactly what this fallback exists to prevent.
+        # These are raw tool-call arguments, so they carry no `id` -- the server
+        # mints it. The sentinel marks the chart as never persisted. Logged because
+        # a turn that called the tool and answered without a chart is otherwise
+        # invisible: the user saw no chart either.
+        _log.warning(
+            "create_adhoc_visualization was called %d time(s) but the answer has no visualization part; "
+            "scoring the last call's arguments",
+            len(acc.adhoc_viz_args),
+        )
         payload["createdVisualizations"] = {
             "objects": [{"id": _ADHOC_VIZ_ID, **acc.adhoc_viz_args[-1]}],
             "reasoning": "\n".join(acc.viz_reasoning_parts),
@@ -447,6 +493,7 @@ class ChatClient:
         preserve_failed: bool = False,
         reasoning_effort: ReasoningEffort | None = None,
         agent_id: str | None = None,
+        user_context: dict[str, Any] | None = None,
     ):
         """Create a chat client bound to one workspace.
 
@@ -455,9 +502,17 @@ class ChatClient:
         entirely and the server keeps its own default. The server honours it only
         while the ``enableGenAiReasoningEffort`` feature flag is on for the
         organization, so setting it is a request rather than a guarantee.
+
+        ``user_context`` is sent as ``userContext`` on every message unless a call passes
+        its own. gen-ai scopes each message by the context it carries and treats a message
+        without one as cleared, so an agentic run's follow-up and clarification turns need
+        it as much as the first question does. A call cannot clear it, since ``None`` falls
+        back to it: a caller that changes or clears the context per message builds the
+        client without one. The server grounds answers in it only while
+        the ``enableAiContextSetup`` feature flag is on for the organization.
         """
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
-        self._auth = {"Authorization": f"Bearer {token}"}
+        self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()}
         # 0/None disables the cap. Also lowered onto the read timeout: the wall-clock check
         # fires between events, so a turn that goes silent needs the transport to give up too.
         budget = _TURN_TIMEOUT_S if turn_timeout_s is None else turn_timeout_s
@@ -474,6 +529,7 @@ class ChatClient:
         self._preserve_failed = preserve_failed
         self._reasoning_effort = normalize_reasoning_effort(reasoning_effort)
         self._agent_id = agent_id
+        self._user_context = user_context
 
     def create_conversation(self) -> str:
         def _do() -> str:
@@ -507,10 +563,12 @@ class ChatClient:
         body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
         if self._reasoning_effort is not None:
             body["options"] = {"reasoningEffort": self._reasoning_effort}
-        # Only when there is one: gen-ai accepts an explicit null, so assigning
-        # unconditionally would quietly change every request that has no attachment.
-        if user_context is not None:
-            body["userContext"] = user_context
+        # A per-call context overrides the client's. Only sent when there is one: gen-ai
+        # accepts an explicit null, so assigning unconditionally would quietly change every
+        # request that has no attachment.
+        context = user_context if user_context is not None else self._user_context
+        if context is not None:
+            body["userContext"] = context
 
         def _do() -> ChatResult:
             # Set fresh on every retry attempt (before opening this attempt's stream, so its

gooddata_eval/core/dataset/langfuse_source.py

@@ -66,10 +66,19 @@ def _user_context_from_raw(raw: dict) -> dict[str, Any] | None:
     item input object or the item metadata. An item carrying an attachment (a WIDGET or
     VIEW descriptor) is meaningless without it: it degrades into a bare question the
     agent has no way to answer, and then fails for a reason that has nothing to do with
-    what the item was written to test. So this has to survive the round trip.
+    what the item was written to test. So this has to survive the round trip, and a
+    value that is not an object raises rather than being skipped, for the same reason.
     """
-    found = _first_of(dict, "user_context", raw.get("input"), raw.get("metadata"))
-    return cast("dict[str, Any]", found) if found is not None else None
+    for source in (raw.get("input"), raw.get("metadata")):
+        if not isinstance(source, dict) or source.get("user_context") is None:
+            continue
+        found = source["user_context"]
+        if not isinstance(found, dict):
+            raise ValueError(
+                f"Langfuse item {raw.get('id')!r}: user_context must be a JSON object, got {type(found).__name__}"
+            )
+        return cast("dict[str, Any]", found)
+    return None
 
 
 def _infer_test_kind(expected_output: object, default: str, metadata: object = None) -> str:

gooddata_eval/core/evaluators/visualization.py

@@ -1,8 +1,10 @@
 # (C) 2026 GoodData Corporation
 """Agentic visualization evaluator — ported from gdc-nas tavern-e2e app/vis_agentic.py."""
 
-from dataclasses import dataclass
+import re
+from dataclasses import dataclass, replace
 from datetime import date
+from typing import Any
 
 from gooddata_eval.core.evaluators.base import ItemEvaluation
 from gooddata_eval.core.models import (
@@ -51,6 +53,14 @@ class EvaluationResult:
     actual_filters: dict[str, list[str]]
     expected_sorts: list[str]
     actual_sorts: list[str]
+    # Whether the agent ran the chart it built (see `execution_signals`). It gates
+    # `strict_pass` only on an item whose expected output sets `requires_execution`: a
+    # chart the user only looks at needs no run, a question that asks for a number does.
+    executed: bool = False
+    requires_execution: bool = False
+    # Whether a number in the reply matches a value the chart returned. None when the reply
+    # states no number. Reported only: reading numbers out of prose has false hits.
+    stated_value_matches: bool | None = None
 
     @property
     def strict_pass(self) -> bool:
@@ -62,6 +72,7 @@ class EvaluationResult:
             and self.filters_correct
             and self.sorts_correct
             and self.viz_type_hard
+            and (self.executed or not self.requires_execution)
         )
 
     @property
@@ -78,6 +89,128 @@ class EvaluationResult:
         )
 
 
+# A number as prose writes it: a sign before or after an optional currency symbol, thousands
+# separators, decimals, then an optional K/M/B scale or a percent sign. Not preceded or
+# followed by a word character, so "Q3", "viz_1" and "2x" are not read as numbers.
+_NUMBER_RE = re.compile(
+    r"(?<![\w.])(?P<sign>[-+]?)[$€£]?(?P<sign2>[-+]?)(?P<int>\d{1,3}(?:,\d{3})+|\d+)(?:\.(?P<frac>\d+))?"
+    r"\s?(?P<suffix>[kKmMbB%])?(?!\w)"
+)
+_SCALES = {"k": 1e3, "m": 1e6, "b": 1e9}
+
+
+def _numbers_in(text: str) -> list[tuple[float, int, str]]:
+    """(value, decimals, suffix) for every number written in ``text``; suffix is lowercased."""
+    out: list[tuple[float, int, str]] = []
+    for m in _NUMBER_RE.finditer(text):
+        integer, fraction = m.group("int").replace(",", ""), m.group("frac") or ""
+        suffix = (m.group("suffix") or "").lower()
+        value = float(f"{integer}.{fraction}" if fraction else integer)
+        if "-" in (m.group("sign"), m.group("sign2")):
+            value = -value
+        out.append((value, len(fraction), suffix))
+    return out
+
+
+def _stated_matches(stated: tuple[float, int, str], raw: list[float], formatted: list[tuple[float, str]]) -> bool:
+    """Whether one number from the reply is a rounding of a returned value."""
+    value, decimals, suffix = stated
+    half_step = 0.5 * 10**-decimals + 1e-9
+    # Same number and same scale: "1.2" does not quote a formatted "1.2M".
+    if any(abs(f - value) < 1e-9 and f_suffix == suffix for f, f_suffix in formatted):
+        return True
+    for v in raw:
+        # A percent states the fraction times 100: "10%" quotes 0.1, and "0.1%" does not.
+        if suffix == "%":
+            if abs(v * 100 - value) <= half_step:
+                return True
+            continue
+        scale = _SCALES.get(suffix, 1.0)
+        if abs(v - value * scale) <= half_step * scale:
+            return True
+    return False
+
+
+def _row_values(data: Any) -> tuple[list[float], list[tuple[float, str]]]:
+    """Numeric cells of an execution result: the raw `rows`, and the (number, scale suffix)
+    pairs read out of `formatted_rows` -- the display strings the agent is told to quote.
+    A malformed result yields nothing rather than raising."""
+    raw: list[float] = []
+    formatted: list[tuple[float, str]] = []
+    if not isinstance(data, dict):
+        return raw, formatted
+    rows, formatted_rows = data.get("rows"), data.get("formatted_rows")
+    for row in rows if isinstance(rows, list) else []:
+        if isinstance(row, dict):
+            raw += [float(v) for v in row.values() if isinstance(v, (int, float)) and not isinstance(v, bool)]
+    for row in formatted_rows if isinstance(formatted_rows, list) else []:
+        if isinstance(row, dict):
+            for v in row.values():
+                if isinstance(v, str):
+                    formatted += [(n[0], n[2]) for n in _numbers_in(v)]
+    return raw, formatted
+
+
+def execution_signals(tool_call_events: list[ToolCallEvent], reply_text: str | None) -> tuple[bool, bool | None]:
+    """``(executed, stated_value_matches)`` for one conversation.
+
+    ``executed`` is True when ``execute_visualization`` succeeded for a ref that a successful
+    ``create_adhoc_visualization`` in the same conversation returned. The chart part of the
+    answer carries no ref, so this cannot tell which of several built charts was shown.
+
+    ``stated_value_matches`` is None when the reply states no number. Otherwise it is True
+    when one of those numbers is a rounding of a value an execution returned, and False when
+    none is -- including when nothing was executed, so the agent could not know the value.
+    """
+    created: set[str] = set()
+    for tc in tool_call_events:
+        if tc.function_name == "create_adhoc_visualization":
+            result = tc.parsed_result()
+            if not isinstance(result, dict):
+                continue
+            ref = result.get("ref") if result.get("status", "success") == "success" else None
+            if isinstance(ref, str):
+                created.add(ref)
+    raw: list[float] = []
+    formatted: list[tuple[float, str]] = []
+    executed = False
+    for tc in tool_call_events:
+        if tc.function_name != "execute_visualization":
+            continue
+        result = tc.parsed_result()
+        args = tc.parsed_arguments()
+        if not isinstance(result, dict) or not isinstance(args, dict):
+            continue
+        ref = args.get("visualization_ref")
+        if result.get("success") is True and isinstance(ref, str) and ref in created:
+            executed = True
+            r, f = _row_values(result.get("data"))
+            raw += r
+            formatted += f
+    stated = _numbers_in(reply_text or "")
+    if not stated:
+        return executed, None
+    return executed, any(_stated_matches(n, raw, formatted) for n in stated)
+
+
+def with_execution(
+    ev: EvaluationResult,
+    tool_call_events: list[ToolCallEvent],
+    reply_text: str | None,
+    requires_execution: bool,
+) -> EvaluationResult:
+    """``ev`` with the execution signals of the conversation that produced it."""
+    executed, stated_value_matches = execution_signals(tool_call_events, reply_text)
+    return replace(
+        ev, executed=executed, requires_execution=requires_execution, stated_value_matches=stated_value_matches
+    )
+
+
+def requires_execution_of(expected_output: object) -> bool:
+    """The item-level `requires_execution` flag; absent or not a boolean reads as False."""
+    return isinstance(expected_output, dict) and expected_output.get("requires_execution") is True
+
+
 def _check_visualization_skill_activated(tool_call_events: list[ToolCallEvent]) -> bool:
     """Return True if set_skills was called with 'visualization' in skill_names."""
     for tc in tool_call_events:
@@ -216,6 +349,14 @@ def evaluation_result_detail(ev: EvaluationResult) -> dict:
         "actual_filters": ev.actual_filters,
         "expected_sorts": ev.expected_sorts,
         "actual_sorts": ev.actual_sorts,
+        # Nested: `quality_score` counts every top-level boolean, and a check an item does not
+        # gate on must not lower it. `executed` joins the top level only when it gates.
+        "execution": {
+            "executed": ev.executed,
+            "required": ev.requires_execution,
+            "stated_value_matches": ev.stated_value_matches,
+        },
+        **({"executed": ev.executed} if ev.requires_execution else {}),
     }
 
 
@@ -227,6 +368,9 @@ class VisualizationEvaluator:
         actual = _extract_actual(chat_result)
         skill_activated = _check_visualization_skill_activated(chat_result.tool_call_events)
         ev, _best_expected = _evaluate_against_candidates(candidates, actual, skill_activated)
+        ev = with_execution(
+            ev, chat_result.tool_call_events, chat_result.text_response, requires_execution_of(item.expected_output)
+        )
         return ItemEvaluation(
             passed=ev.strict_pass,
             rank_key=(ev.strict_pass, ev.strict_checks_passed_count),

{gooddata_eval-1.76.1.dev1.dist-info → gooddata_eval-1.76.1.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.76.1.dev1
+Version: 1.76.1.dev3
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.76.1.dev1
+Requires-Dist: gooddata-sdk~=1.76.1.dev3
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15
@@ -603,6 +603,55 @@ The `expected_output` rubric:
 Each criterion is scored independently by the LLM judge, so `quality_score`
 is the fraction of satisfied criteria.
 
+### Items asked in a dashboard context (`user_context`)
+
+An item can be asked the way a user asks from an open dashboard or an attached widget.
+Its `user_context` is sent verbatim as `userContext` on every chat message of the item,
+follow-up and clarification messages included — the server treats a message without one
+as a cleared context. Requires the `enableAiContextSetup` feature flag on the target
+organization — without it the server ignores the dashboard view. Applies to chat items
+only; `dashboard_summary` items carry their scope in `summary_input` instead. No
+`userContext` key is sent at all when neither the item nor, for `agentic_conversation`,
+the current turn sets one.
+
+```json
+{
+  "id": "ctx-001",
+  "dataset_name": "my_dataset_ctx",
+  "test_kind": "general_question",
+  "question": "Which dashboard am I looking at, and what does it cover?",
+  "user_context": {
+    "view": {
+      "dashboard": {
+        "id": "sales_overview",
+        "title": "Sales Overview",
+        "widgets": [
+          {"widgetType": "insight", "widgetId": "w1", "title": "Revenue by Month", "visualizationId": "revenue_by_month"}
+        ]
+      }
+    }
+  },
+  "expected_output": "Names the Sales Overview dashboard and summarizes its charts."
+}
+```
+
+The schema belongs to the AI chat API (`UserContext`: `view.dashboard`,
+`referencedObjects`, `activeObject`), so it is not validated here beyond being an object.
+In a Langfuse dataset, put it in the item `metadata` (or in the `input` object) under
+`user_context`. An object in `input` wins and `metadata` is then not read; a missing or
+`null` value in `input` falls back to `metadata`. The value used must be an object or `null`;
+anything else fails the dataset load.
+
+`agentic_conversation` items take the item's `user_context` as the context of the first
+turn. A turn in `expected_output.turns` can set its own `user_context`, which applies
+from that turn on until another turn changes it, as the attached context does in the UI:
+
+| Turn | Context sent |
+|---|---|
+| no `user_context` key | the current one, unchanged |
+| `"user_context": {...}` | this one, from this turn on |
+| `"user_context": null` | none, from this turn on |
+
 ## Supported test kinds
 
 | test_kind | What the agent must produce | Extra required |

{gooddata_eval-1.76.1.dev1.dist-info → gooddata_eval-1.76.1.dev3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 gooddata_eval/__init__.py,sha256=9Nz0iJXzkJWiL40PijM6AMpAqQOH-dbYNfDtDysBPYc,186
 gooddata_eval/_version.py,sha256=L4nT6RJQ2vc_F9stvO59xZv92xsRUqbpUh1sGlb-UNU,195
 gooddata_eval/cli/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
-gooddata_eval/cli/agentic_runner.py,sha256=9_yvTGEoYgbsZfwn9pnVY9Hv2qEBEdYhUrTLiMbv7FA,23638
+gooddata_eval/cli/agentic_runner.py,sha256=hAihzRS0LCTkRRggyboOp7dXd5HoehoFU8LpBU6PJ1c,24258
 gooddata_eval/cli/main.py,sha256=ry-XbCcNEb6Obm5hfiRo6etGY9aQvFT9UbAt_pPmbA0,32919
 gooddata_eval/core/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/_output.py,sha256=sFbNtKg1lx431jpb15jISLlOXGTZ-r_KodfaTVyka6I,580
@@ -19,24 +19,24 @@ gooddata_eval/core/agentic/_conversation_context.py,sha256=WzF_HJzf3fvDJ7EfvekBu
 gooddata_eval/core/agentic/_gate.py,sha256=irI4eWOCfmsbKd6l-HdpoAlnWkq2wjo4b5crGU_Er1o,2631
 gooddata_eval/core/agentic/_langfuse.py,sha256=7V1PFQbWpRnFz0a3wklcI-ROYjGC1wGQv2QCfQqpdws,22604
 gooddata_eval/core/agentic/_trace_linker.py,sha256=iXMTxIXUsF5WNSen7v_K8JJ2j1_37lh-2VAGqQxxmkI,13538
-gooddata_eval/core/agentic/alert_skill.py,sha256=CSwTR9OdlI6rUp7J4Wo0rDq0fK8mY57hJ3w5Q2Wpeds,45287
-gooddata_eval/core/agentic/anomaly_detection.py,sha256=tV7TB4RPELSMzpWaiW87JEkYndTO8aeAe5qcQzkpCmk,26538
-gooddata_eval/core/agentic/conversation.py,sha256=qR278ztVm2lzA9ntjDiIzNiELBYi5uCsWRh1-ifxKNs,57625
-gooddata_eval/core/agentic/dashboard_skill.py,sha256=4lu7v6VeIR_h9FUFBrpD57upA0Rd2Kox1ZP_X-7jQaQ,51854
-gooddata_eval/core/agentic/general_question.py,sha256=Jp3FaS4Tvmfo0dx9iceiZ3Fr1QzJKDl7fVZZDetS22I,14684
-gooddata_eval/core/agentic/guardrail.py,sha256=rYpnsun9mLOSYucWfg1dbdrpbzQcz2V3eLwgII8dD0w,13145
-gooddata_eval/core/agentic/kda_skill.py,sha256=-XP6DeWz8XFzAxcvWyMFWy5aVM2_ZhCliF73VXV-EbU,24256
-gooddata_eval/core/agentic/metric_skill.py,sha256=778uBZvJ8bFTAO1N4nOj7rKoln0JhzTh_rX5bZD89mw,25779
-gooddata_eval/core/agentic/report_skill.py,sha256=fwxhoOaDrMc9CjbO5ANpgLSXqYUcUPUFTMxkVcOVfhk,34254
-gooddata_eval/core/agentic/search_tool.py,sha256=qZDYpYFLf4-TbpEgVCgAYEUruR9_FG1AYLTKwS8Klbk,11038
-gooddata_eval/core/agentic/visualization.py,sha256=s81CgeVKAm2FUa9Fsqk9x90k4kMjny-y9dEQab245S0,24331
-gooddata_eval/core/agentic/what_if.py,sha256=V_4D29pIdFO2wBUgtF0-85oiK6gBWgpN3BfPXBLW24g,24526
+gooddata_eval/core/agentic/alert_skill.py,sha256=KLHazw98foUgCgeeIKH5Q_C5hXIr3j-GqbGZF1UZCEM,45466
+gooddata_eval/core/agentic/anomaly_detection.py,sha256=ckt4Y3kOUmW1EArfJPbUtvSNUz_RmzAIXo-iIK4wvmc,26717
+gooddata_eval/core/agentic/conversation.py,sha256=rssoR0DlFvW6MMeddfP4teL5zh3UpfZw4fFGbk889jw,59199
+gooddata_eval/core/agentic/dashboard_skill.py,sha256=4ITF44SYzonRhKjGYmcsa-3Bf-YjNohzgw7NWxhIjWI,52033
+gooddata_eval/core/agentic/general_question.py,sha256=742pP3ILGjrcmHWj2mS5vmoop_IxyHuYidqggy_iBU0,14591
+gooddata_eval/core/agentic/guardrail.py,sha256=IfCzsRslwAfsEWnhVwwXBMezQPGhN6YCpu8wZ2ofAGo,13324
+gooddata_eval/core/agentic/kda_skill.py,sha256=ZP6_Xikk0LPwW5yCYvaePqHAy4lpnb3DxwslUo83LhM,24435
+gooddata_eval/core/agentic/metric_skill.py,sha256=PHT09IKpEYQX2d_CkJYv_ZuMywOcrijfMXXPlLfUev8,28249
+gooddata_eval/core/agentic/report_skill.py,sha256=TLoGYmRDkHy2jXcmMbIDwXlohWTXbhN9nuL5uRRJBZE,34433
+gooddata_eval/core/agentic/search_tool.py,sha256=k1rdY7lX0bQr68PlO6-B1RcGbWwtUwygZ4nrGmgkclA,11217
+gooddata_eval/core/agentic/visualization.py,sha256=4g1nu-F7H54nttIdpmlBfipr6FS8m-J4jtilktGtYhc,26137
+gooddata_eval/core/agentic/what_if.py,sha256=-1eJAvYiZdraTjb1m03JlMtJwQmyel52v2vGQEVd2Ts,24705
 gooddata_eval/core/chat/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/chat/render.py,sha256=CqsKN8__DlaiJj_bmcPZ1poc5LfUqtz7EzKRDbks4gU,2082
-gooddata_eval/core/chat/sse_client.py,sha256=gj0GeVeVNMbqZh-CexhWTXwoyRxiFWs9wp8Xr_0QNMU,26337
+gooddata_eval/core/chat/sse_client.py,sha256=jvNAu1EXlc1ViqHJ1NC0C99A05An8QS7MMPj63_Wnw4,29420
 gooddata_eval/core/dataset/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/dataset/from_insights.py,sha256=QxVprksX-l1Z4yLiOQOK8ZaUK5z64SCjdQym3cip99s,40520
-gooddata_eval/core/dataset/langfuse_source.py,sha256=s4HvKYP2lOPLp2yFtM5zjIU0Sec2aDwwYcwZH5CXijg,6910
+gooddata_eval/core/dataset/langfuse_source.py,sha256=6-nwFHbzVLEESkxEZoYPgDWRcPoXkohaYcdVKpxIRAY,7306
 gooddata_eval/core/dataset/local.py,sha256=qoFFtRx_bNwlT1U37o8CKkSPwCad3JpmHPcNRuGSbjE,1233
 gooddata_eval/core/evaluators/__init__.py,sha256=luJlcrwMVuyyr2I8ohfNPnlRHRa_cl-Kb19jRe7eEyQ,3130
 gooddata_eval/core/evaluators/_deep_subset.py,sha256=TonPUfkUZL4q1a9t3cczDC2vSF54ynEDDQRxP6p-fvc,1407
@@ -51,7 +51,7 @@ gooddata_eval/core/evaluators/guardrail.py,sha256=J3sX0v_ImbV-GC3c_vMtQrn9Sn0sy-
 gooddata_eval/core/evaluators/metric_skill.py,sha256=ckQvo-TX_bwGWcTkxaHMztMcg7lYBegH5uFOK4wfNSo,3005
 gooddata_eval/core/evaluators/search_tool.py,sha256=c6iK_tHMaDaUjYuo5EnAYDGe53gTDTQtUl226hZq0JY,3070
 gooddata_eval/core/evaluators/summary.py,sha256=aBEW62nCbjKjdkHzLJnqhrPFz0DkJIlJDSYCd8iP7nA,7021
-gooddata_eval/core/evaluators/visualization.py,sha256=EEWEEpLl3lKlFTyyL4UAtkAwjgDx1vg_ZMNO3ByCvzs,9381
+gooddata_eval/core/evaluators/visualization.py,sha256=lFHbReZh1DnS0Bgx6iTgRbeN7YZWoKxGvrkCgAelgEw,16194
 gooddata_eval/core/langfuse/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/langfuse/_env.py,sha256=UMXQblsVleWE7KbKu_mqRXdctss7Hh2eQVAmyFT2yIA,1322
 gooddata_eval/core/langfuse/client.py,sha256=xyZWGRwzloFQVWPcKU_PAC74DU0hCCHKj-J0nybl5xI,7802
@@ -66,8 +66,8 @@ gooddata_eval/core/reporting/json_report.py,sha256=q3Jirc_Zhvzg5lkG_kM6JdnViNLe6
 gooddata_eval/core/reporting/report_template.html,sha256=vLkl95OZsm3dMdijv5zda-15wYPE5UvTs4uKeWnjCiI,22332
 gooddata_eval/core/summary/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/summary/http_client.py,sha256=dPArJ6zoCu1W9xmYXFA1WMDNsSOkhn3gaCpYgCUp3gk,2174
-gooddata_eval-1.76.1.dev1.dist-info/METADATA,sha256=lO3G-2EYRBjp4OWTb0xBHHfJ5QI1t9mv5TBGihlNi9U,35069
-gooddata_eval-1.76.1.dev1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-gooddata_eval-1.76.1.dev1.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
-gooddata_eval-1.76.1.dev1.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
-gooddata_eval-1.76.1.dev1.dist-info/RECORD,,
+gooddata_eval-1.76.1.dev3.dist-info/METADATA,sha256=SwIoGDdCBtQkcQvZXliFCJ7eVqeMmTKVQjwyad8jDPE,37256
+gooddata_eval-1.76.1.dev3.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+gooddata_eval-1.76.1.dev3.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
+gooddata_eval-1.76.1.dev3.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
+gooddata_eval-1.76.1.dev3.dist-info/RECORD,,