gooddata-eval 1.72.1.dev3__py3-none-any.whl → 1.72.1.dev5__py3-none-any.whl

gooddata_eval/core/agentic/alert_skill.py

@@ -250,7 +250,7 @@ def generate_simulated_alert_response(
     response = openai_client.chat.completions.create(
         model="gpt-4o",
         messages=messages,
-        temperature=0.5,
+        temperature=0,
     )
     return response.choices[0].message.content or ""
 

gooddata_eval/core/agentic/conversation.py

@@ -252,7 +252,7 @@ def _get_sim_user_response(agent_message: str, turn: TurnDefinition, expected_ou
                         ),
                     },
                 ],
-                temperature=0.5,
+                temperature=0,
             )
             content = response.choices[0].message.content
             return content.strip() if content else "Please proceed with sensible defaults."

gooddata_eval/core/agentic/kda_skill.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 
 import logging
 import os
-import re
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
@@ -15,36 +14,79 @@ from gooddata_eval.core.models import ToolCallEvent
 _log = logging.getLogger(__name__)
 
 _DEFAULT_K = 1
-# Disambiguation safety net only (create+execute always run together in the same
-# turn) -- 3 covers metric and period each needing their own clarifying question.
-_DEFAULT_MAX_ITERATIONS = 3
+# Disambiguation safety net only (create+execute always run together in the same turn) --
+# 3 real questions' worth (metric, period, +1 slack) since a simulated reply is now sent on
+# every non-final turn (see run_agentic_kda_skill), not just ones classified as a question.
+_DEFAULT_MAX_ITERATIONS = 4
 
 
-def _is_asking_kda_clarification(text: str) -> bool:
-    """True if ``text`` reads as the agent asking for input, not a final answer.
-
-    KDA-specific, not shared with metric_skill.py/conversation.py -- each skill's
-    disambiguation heuristic has already drifted independently. Requires the text to
-    end on "?" (a "?" anywhere also matches a final answer that merely quotes one).
+def _build_period_hint(expected_output: dict) -> str | None:
+    """Build a period hint from whichever of expected_output's Date Attribute/Analyzed
+    Period/Reference Period are present -- a question about only one of them (e.g. "which
+    date dimension?") must still get an answerable hint, not None just because the other
+    two are absent.
+    """
+    date_attr = expected_output.get("Date Attribute")
+    analyzed = expected_output.get("Analyzed Period")
+    reference_period = expected_output.get("Reference Period")
+    if not (date_attr or analyzed or reference_period):
+        return None
+    parts = []
+    if date_attr:
+        parts.append(date_attr)
+    if analyzed and reference_period:
+        parts.append(f"comparing {analyzed} to {reference_period}")
+    elif analyzed:
+        parts.append(f"period {analyzed}")
+    elif reference_period:
+        parts.append(f"compared to {reference_period}")
+    return ", ".join(parts)
+
+
+def _build_clarification_prompt(
+    agent_message: str, measure_candidates: dict | list[dict] | None, period_hint: str | None
+) -> str:
+    """Build the simulated-user prompt, referencing only whatever candidates/period-hint
+    are actually usable -- an empty/None candidate must drop the "acceptable metric/fact"
+    clause entirely rather than assert a literal "None" as if it were a real option.
     """
-    if not text:
-        return False
-    t = text.strip().lower()
-    if t.endswith("?"):
-        return True
-    # "To clarify, ..." means "in other words" (a final answer), not a request for one --
-    # strip it first so "clarif" below only matches genuine clarification requests.
-    t = re.sub(r"^(just )?to clarify,?\s*", "", t)
-    return "could you" in t or "please provide" in t or "clarif" in t
-
-
-def generate_simulated_kda_response(agent_message: str, measure_candidates: dict | list[dict] | None) -> str:
+    candidates = [
+        c for c in (measure_candidates if isinstance(measure_candidates, list) else [measure_candidates]) if c
+    ]
+    reference = ""
+    if candidates:
+        candidate_desc = "; or ".join(
+            f"{c.get('type')} '{c.get('id')}'" + (f" (aggregation {c['aggregation']})" if c.get("aggregation") else "")
+            for c in candidates
+        )
+        reference = f"an acceptable metric/fact is {candidate_desc}"
+    if period_hint:
+        reference = (
+            f"{reference}; the intended time period is {period_hint}"
+            if reference
+            else f"the intended time period is {period_hint}"
+        )
+    return (
+        f"You are simulating a user in a conversation with a BI assistant that runs key driver "
+        f"analysis. The assistant asked: '{agent_message}'. "
+        + (f"For reference, {reference}. " if reference else "")
+        + "Reply briefly as the user, answering whichever of those the assistant actually asked about."
+    )
+
+
+def generate_simulated_kda_response(
+    agent_message: str,
+    measure_candidates: dict | list[dict] | None,
+    period_hint: str | None = None,
+) -> str:
     """Generate a user reply to keep the KDA-skill conversation going (gpt-4o-mini).
 
-    Used only when the agent asks a clarifying question instead of triggering KDA
-    directly. Picks *any* candidate from ``measure_candidates`` -- scope only needs KDA
-    to trigger, not the resulting measure to be exactly right. Always OpenAI regardless
-    of the combo's own provider -- this is test-harness plumbing, not the system under test.
+    Called on any turn that didn't trigger KDA, whatever the agent's response actually
+    said -- most often a clarifying question about the measure, the period, or both, so
+    both are given as reference and the reply answers whichever was actually asked.
+    Scope only needs KDA to trigger, not the resulting measure/period to be exactly
+    right. Always OpenAI regardless of the combo's own provider -- this is
+    test-harness plumbing, not the system under test.
     """
     try:
         from openai import OpenAI  # noqa: PLC0415
@@ -56,21 +98,12 @@ def generate_simulated_kda_response(agent_message: str, measure_candidates: dict
         raise OSError("OPENAI_API_KEY environment variable is not set")
 
     client = OpenAI(api_key=api_key)
-    candidates = measure_candidates if isinstance(measure_candidates, list) else [measure_candidates or {}]
-    candidate_desc = "; or ".join(
-        f"{c.get('type')} '{c.get('id')}'" + (f" (aggregation {c['aggregation']})" if c.get("aggregation") else "")
-        for c in candidates
-    )
-    prompt = (
-        f"You are simulating a user in a conversation with a BI assistant that runs key driver "
-        f"analysis. The assistant said: '{agent_message}'. "
-        f"The user is happy to proceed with any of the following: {candidate_desc}. "
-        f"Reply briefly as the user, picking whichever of those the assistant offered."
-    )
+    prompt = _build_clarification_prompt(agent_message, measure_candidates, period_hint)
     response = client.chat.completions.create(
         model="gpt-4o-mini",
         messages=[{"role": "user", "content": prompt}],
         max_tokens=150,
+        temperature=0,
         timeout=30,
     )
     return response.choices[0].message.content or "Please proceed with either option."
@@ -171,9 +204,12 @@ def run_agentic_kda_skill(
 
     Each run is normally one message, one turn -- create and execute are always called
     together in the same turn (the skill's own system prompt: "NO confirmation needed").
-    The only thing that can extend a run up to ``max_iterations`` turns is the agent
-    asking a clarifying question instead of triggering KDA directly; a simulated user
-    reply nudges it forward.
+    A run only extends past turn 1, up to ``max_iterations``, when the agent's response
+    has no create call and isn't empty; a simulated user reply is then always sent, with
+    no attempt to classify whether the text was actually asking for input (matching
+    visualization.py/alert_skill.py's own break conditions) -- missing a genuine
+    clarifying question hard-fails the run, while sending one after an unrecognized final
+    answer only costs one harmless extra turn, so the asymmetry favors never guessing.
     """
     if k < 1:
         # k=0 or negative would otherwise silently run once, indistinguishable from k=1.
@@ -212,17 +248,22 @@ def run_agentic_kda_skill(
                 # execute tool isn't available at all when data-sharing is off for the org).
                 turn_wall_clock_sec = chat_result.turn_wall_clock_sec
                 break
+            if not response_text:
+                break
             if iteration >= max_iterations - 1:
                 break
-            if _is_asking_kda_clarification(response_text):
-                measure_candidates = expected_output.get("Measure") if isinstance(expected_output, dict) else None
-                try:
-                    current_question = generate_simulated_kda_response(response_text, measure_candidates)
-                    disambiguated = True
-                except Exception as exc:  # noqa: BLE001 -- safety net, not the assertion; end only this run
-                    _log.warning("Simulated KDA user reply failed for conversation %s: %s", conv_id, exc)
-                    break
-            else:
+            # No text classification -- matches visualization.py/alert_skill.py: break only on
+            # the goal signal (create_args set) or an empty response, otherwise always send a
+            # simulated reply. A false positive (agent had already given a final answer) costs
+            # one harmless extra turn; a false negative (missing a genuine clarifying question)
+            # would hard-fail the run, so the asymmetry favors never trying to tell them apart.
+            measure_candidates = expected_output.get("Measure") if isinstance(expected_output, dict) else None
+            period_hint = _build_period_hint(expected_output) if isinstance(expected_output, dict) else None
+            try:
+                current_question = generate_simulated_kda_response(response_text, measure_candidates, period_hint)
+                disambiguated = True
+            except Exception as exc:  # noqa: BLE001 -- safety net, not the assertion; end only this run
+                _log.warning("Simulated KDA user reply failed for conversation %s: %s", conv_id, exc)
                 break
 
         ev = _evaluate_run(create_args, execute_result, turn_completed, disambiguated)

gooddata_eval/core/agentic/metric_skill.py

@@ -95,6 +95,7 @@ def generate_simulated_response(agent_message: str, expected_output: dict) -> st
         model="gpt-4o-mini",
         messages=[{"role": "user", "content": prompt}],
         max_tokens=150,
+        temperature=0,
     )
     return response.choices[0].message.content or "Please proceed."
 

gooddata_eval/core/agentic/visualization.py

@@ -141,7 +141,7 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu
                 ),
             },
         ],
-        temperature=0.5,
+        temperature=0,
     )
     content = response.choices[0].message.content
     return content.strip() if content else ""

{gooddata_eval-1.72.1.dev3.dist-info → gooddata_eval-1.72.1.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.72.1.dev3
+Version: 1.72.1.dev5
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.72.1.dev3
+Requires-Dist: gooddata-sdk~=1.72.1.dev5
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.72.1.dev3.dist-info → gooddata_eval-1.72.1.dev5.dist-info}/RECORD

@@ -13,14 +13,14 @@ gooddata_eval/core/workspace.py,sha256=S8exiy5gpDtcRtT8-i2fIRALF34Pbo0J46xm8X_ld
 gooddata_eval/core/agentic/__init__.py,sha256=M35wdSW7Bw5vidfHN4MuL0ZR8L7igUnF5mNWiboglcw,3030
 gooddata_eval/core/agentic/_catalog.py,sha256=eCaeyvLF-FCN0J3UnXSHaKtufU6HvrM2mxnmgyXsY1U,2085
 gooddata_eval/core/agentic/_langfuse.py,sha256=nRI_tcmORBWbuF3AJKAL8VX-mFkLyZBQXagCGSQ6xIE,15102
-gooddata_eval/core/agentic/alert_skill.py,sha256=Be7rf1sfToQfO_pzg1BwXJcC2MwlY01Vwg8egCxjREI,27226
-gooddata_eval/core/agentic/conversation.py,sha256=Yglf68PZ_Yb69z9ujTfX7HQcUrOzwlB85vnpul9SLMk,19073
+gooddata_eval/core/agentic/alert_skill.py,sha256=xFYOmpDX5Qrjj0cVMilf99D7DfLHeVwMyrSftKB1f_c,27224
+gooddata_eval/core/agentic/conversation.py,sha256=c0vj5nk1IsrHFiMAje4997sXo3vayk9oTTv5nkk-D6s,19071
 gooddata_eval/core/agentic/general_question.py,sha256=Tw2M-FJJWDo0O4leD5AoYW8UBpVQWSZrZJ0fNbHTHW8,8364
 gooddata_eval/core/agentic/guardrail.py,sha256=qE9vADYilDnmqjcwEE-HGF3XsOcGjdvsKFfLSX9IFYQ,8251
-gooddata_eval/core/agentic/kda_skill.py,sha256=d0lY0nUvb9D0Lu-bI_E55EecUFJnnqRS9EkUEaDVJBI,15837
-gooddata_eval/core/agentic/metric_skill.py,sha256=zZs8cRcFdDLIV_rY6wQKd9o-0VbQYK6EOL8KbmZpk4A,14117
+gooddata_eval/core/agentic/kda_skill.py,sha256=XsKPhhEHQGJTPC3AufOgantxDTdFUwpjLpqlMfPLLEk,17974
+gooddata_eval/core/agentic/metric_skill.py,sha256=0KmbVliieyL6p3z9AGmMDgMlBDBQlI1oBp1XOgZ-M8U,14140
 gooddata_eval/core/agentic/search_tool.py,sha256=L6yEs0hJymC_beYmily9ivw3M6tjLgDnF9_6kr7g11s,8020
-gooddata_eval/core/agentic/visualization.py,sha256=rZPN6CF_XpMsmPHSwzUP4iisLBL_99g2UIjIww_GbbQ,17129
+gooddata_eval/core/agentic/visualization.py,sha256=qhwJ06_SauPjOl1yJ3teZmf42YQjpcSwTy3yuc3kz30,17127
 gooddata_eval/core/chat/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/chat/sse_client.py,sha256=oYLxMWy6Hy0yoC8InnTkKLWN4Rsg1gSSIiE0FzaVa4o,16062
 gooddata_eval/core/dataset/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
@@ -45,8 +45,8 @@ gooddata_eval/core/reporting/console.py,sha256=kCHt4sCSaSRh0ZWqaSEJOl6_Jgb6uiouS
 gooddata_eval/core/reporting/json_report.py,sha256=sKPG9-uxzDsf6EJyBXoaHVIfN1g-E2rZ8fA0wtyBYb4,2943
 gooddata_eval/core/summary/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/summary/http_client.py,sha256=dPArJ6zoCu1W9xmYXFA1WMDNsSOkhn3gaCpYgCUp3gk,2174
-gooddata_eval-1.72.1.dev3.dist-info/METADATA,sha256=ykwpTF22guapOxGKX7IHM8mZTAncsM7vZf1wCqYLBWM,9995
-gooddata_eval-1.72.1.dev3.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-gooddata_eval-1.72.1.dev3.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
-gooddata_eval-1.72.1.dev3.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
-gooddata_eval-1.72.1.dev3.dist-info/RECORD,,
+gooddata_eval-1.72.1.dev5.dist-info/METADATA,sha256=8-k1gfi50ned3R52_nOj99ArVhTM-HMdhnD4CJh9HbI,9995
+gooddata_eval-1.72.1.dev5.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+gooddata_eval-1.72.1.dev5.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
+gooddata_eval-1.72.1.dev5.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
+gooddata_eval-1.72.1.dev5.dist-info/RECORD,,