gooddata-eval 1.72.1.dev4__py3-none-any.whl → 1.72.1.dev6__py3-none-any.whl

gooddata_eval/core/agentic/alert_skill.py

@@ -250,7 +250,7 @@ def generate_simulated_alert_response(
     response = openai_client.chat.completions.create(
         model="gpt-4o",
         messages=messages,
-        temperature=0.5,
+        temperature=0,
     )
     return response.choices[0].message.content or ""
 

gooddata_eval/core/agentic/conversation.py

@@ -25,6 +25,8 @@ from gooddata_eval.core.scoring import (
 
 _REF_PATTERN = re.compile(r"\$ref:([\w_]+)\.([\w_]+)")
 
+_DEFAULT_MAX_CLARIFICATION_TURNS = 7
+
 
 class TurnDefinition(BaseModel):
     """Definition of a single turn in a multi-turn conversation evaluation."""
@@ -192,13 +194,6 @@ def _check_output_correct(turn: TurnDefinition, chat_result: ChatResult) -> bool
     return None
 
 
-def _is_asking_clarification(text: str) -> bool:
-    if not text:
-        return False
-    t = text.lower()
-    return "?" in t or "could you" in t or "please" in t or "clarif" in t
-
-
 def _get_sim_user_response(agent_message: str, turn: TurnDefinition, expected_output: dict | None) -> str:
     """Generate a simulated user reply to an agent clarification question."""
     otype = turn.expected_output_type
@@ -252,7 +247,7 @@ def _get_sim_user_response(agent_message: str, turn: TurnDefinition, expected_ou
                         ),
                     },
                 ],
-                temperature=0.5,
+                temperature=0,
             )
             content = response.choices[0].message.content
             return content.strip() if content else "Please proceed with sensible defaults."
@@ -277,7 +272,7 @@ def run_agentic_conversation(
     token: str,
     workspace_id: str,
     fixture: ConversationFixture,
-    max_clarification_turns: int = 20,
+    max_clarification_turns: int = _DEFAULT_MAX_CLARIFICATION_TURNS,
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
 ) -> ConversationResult:
@@ -307,8 +302,22 @@ def run_agentic_conversation(
             owns_conversation = True
 
         for turn in fixture.turns:
-            # Resolve $ref placeholders using outputs captured from prior turns.
-            resolved_expected = _resolve_refs(turn.expected_output, turn_outputs)
+            try:
+                resolved_expected = _resolve_refs(turn.expected_output, turn_outputs)
+            except ValueError as exc:
+                print(f"[SKIP] turn '{turn.turn_id}': {exc}")
+                turn_results.append(
+                    TurnResult(
+                        turn_id=turn.turn_id,
+                        expected_skill=turn.expected_skill,
+                        skill_routing=False,
+                        output_present=False,
+                        no_error=False,
+                        activated_skills=[],
+                        output_correct=False,
+                    )
+                )
+                continue
             resolved_turn = turn.model_copy(update={"expected_output": resolved_expected})
 
             clarification_turns = 0
@@ -327,13 +336,13 @@ def run_agentic_conversation(
                 response_text = (chat_result.text_response or "").strip()
                 if not response_text and chat_result.alert_proposals:
                     response_text = render_alert_proposal(chat_result.alert_proposals[-1])
-                asking = _is_asking_clarification(response_text) or bool(chat_result.alert_proposals)
-                if asking and clarification_turns < max_clarification_turns:
-                    clarification_turns += 1
-                    total_clarification_turns += 1
-                    current_message = _get_sim_user_response(response_text, resolved_turn, resolved_expected)
-                else:
+                if not response_text and not chat_result.tool_call_events:
+                    break
+                if clarification_turns >= max_clarification_turns:
                     break
+                clarification_turns += 1
+                total_clarification_turns += 1
+                current_message = _get_sim_user_response(response_text, resolved_turn, resolved_expected)
 
             activated = _activated_skills(all_tool_calls)
             skill_routing = turn.expected_skill in activated if activated else False
@@ -397,7 +406,7 @@ def evaluate_agentic_conversation(
     token: str,
     workspace_id: str,
     fixture: ConversationFixture,
-    max_clarification_turns: int = 20,
+    max_clarification_turns: int = _DEFAULT_MAX_CLARIFICATION_TURNS,
     initial_conversation_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",

gooddata_eval/core/agentic/kda_skill.py

@@ -103,6 +103,7 @@ def generate_simulated_kda_response(
         model="gpt-4o-mini",
         messages=[{"role": "user", "content": prompt}],
         max_tokens=150,
+        temperature=0,
         timeout=30,
     )
     return response.choices[0].message.content or "Please proceed with either option."

gooddata_eval/core/agentic/metric_skill.py

@@ -72,16 +72,29 @@ def _best_maql_match(actual_maql: str, expected_outputs: list[dict]) -> tuple[bo
     return False, expected_outputs[0].get("maql", "") if expected_outputs else ""
 
 
+class SimulatedResponseError(RuntimeError):
+    """The simulated user could not reply: openai missing, no API key, or the provider failed.
+
+    Carries every expected setup/provider failure so callers can end the run without
+    swallowing programming errors raised from the same call.
+    """
+
+
 def generate_simulated_response(agent_message: str, expected_output: dict) -> str:
-    """Generate a user reply to keep the metric-skill conversation going (gpt-4o-mini)."""
+    """Generate a user reply to keep the metric-skill conversation going (gpt-4o-mini).
+
+    Raises:
+        SimulatedResponseError: openai is not installed, OPENAI_API_KEY is unset, or the
+            provider call failed.
+    """
     try:
-        from openai import OpenAI  # noqa: PLC0415
+        from openai import OpenAI, OpenAIError  # noqa: PLC0415
     except ImportError as exc:
-        raise RuntimeError("openai package is required for generate_simulated_response") from exc
+        raise SimulatedResponseError("openai package is required for generate_simulated_response") from exc
 
     api_key = os.environ.get("OPENAI_API_KEY")
     if not api_key:
-        raise OSError("OPENAI_API_KEY environment variable is not set")
+        raise SimulatedResponseError("OPENAI_API_KEY environment variable is not set")
 
     client = OpenAI(api_key=api_key)
     expected_maql = expected_output.get("maql", "")
@@ -91,11 +104,15 @@ def generate_simulated_response(agent_message: str, expected_output: dict) -> st
         f"The user originally asked to create a metric with MAQL: {expected_maql}. "
         f"Reply briefly as the user, providing any clarification the assistant needs."
     )
-    response = client.chat.completions.create(
-        model="gpt-4o-mini",
-        messages=[{"role": "user", "content": prompt}],
-        max_tokens=150,
-    )
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=150,
+            temperature=0,
+        )
+    except OpenAIError as exc:
+        raise SimulatedResponseError(f"simulated user reply failed: {exc}") from exc
     return response.choices[0].message.content or "Please proceed."
 
 
@@ -165,13 +182,6 @@ def _delete_metric(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> None:
         print(f"[CLEANUP] Failed to delete metric {metric_id}: {exc}")
 
 
-def _is_asking_clarification(text: str) -> bool:
-    if not text:
-        return False
-    t = text.lower()
-    return "?" in t or "could you" in t or "please provide" in t or "clarif" in t
-
-
 def _execute_single_metric_run(
     client: ChatClient,
     sdk: GoodDataSdk,
@@ -203,9 +213,14 @@ def _execute_single_metric_run(
                 metric_id_to_delete = candidate.get("metric_id")
                 break
             response_text = (chat_result.text_response or "").strip()
-            if _is_asking_clarification(response_text):
+            if not response_text and not chat_result.tool_call_events:
+                break
+            if _iteration >= max_iterations - 1:
+                break
+            try:
                 current_question = generate_simulated_response(response_text, primary_expected)
-            else:
+            except SimulatedResponseError as exc:
+                print(f"[SIM-USER] Simulated reply failed for conversation {conversation_id}: {exc}")
                 break
 
         actual_maql = (metric_result or {}).get("maql", "")

gooddata_eval/core/agentic/visualization.py

@@ -141,7 +141,7 @@ def generate_simulated_response(agent_message: str, expected_output: CreatedVisu
                 ),
             },
         ],
-        temperature=0.5,
+        temperature=0,
     )
     content = response.choices[0].message.content
     return content.strip() if content else ""

{gooddata_eval-1.72.1.dev4.dist-info → gooddata_eval-1.72.1.dev6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.72.1.dev4
+Version: 1.72.1.dev6
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.72.1.dev4
+Requires-Dist: gooddata-sdk~=1.72.1.dev6
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.72.1.dev4.dist-info → gooddata_eval-1.72.1.dev6.dist-info}/RECORD

@@ -13,14 +13,14 @@ gooddata_eval/core/workspace.py,sha256=S8exiy5gpDtcRtT8-i2fIRALF34Pbo0J46xm8X_ld
 gooddata_eval/core/agentic/__init__.py,sha256=M35wdSW7Bw5vidfHN4MuL0ZR8L7igUnF5mNWiboglcw,3030
 gooddata_eval/core/agentic/_catalog.py,sha256=eCaeyvLF-FCN0J3UnXSHaKtufU6HvrM2mxnmgyXsY1U,2085
 gooddata_eval/core/agentic/_langfuse.py,sha256=nRI_tcmORBWbuF3AJKAL8VX-mFkLyZBQXagCGSQ6xIE,15102
-gooddata_eval/core/agentic/alert_skill.py,sha256=Be7rf1sfToQfO_pzg1BwXJcC2MwlY01Vwg8egCxjREI,27226
-gooddata_eval/core/agentic/conversation.py,sha256=Yglf68PZ_Yb69z9ujTfX7HQcUrOzwlB85vnpul9SLMk,19073
+gooddata_eval/core/agentic/alert_skill.py,sha256=xFYOmpDX5Qrjj0cVMilf99D7DfLHeVwMyrSftKB1f_c,27224
+gooddata_eval/core/agentic/conversation.py,sha256=kAXkPfF1pYSaZ4HooapqIqVJ52H4BO6u-uowoblqMr8,19442
 gooddata_eval/core/agentic/general_question.py,sha256=Tw2M-FJJWDo0O4leD5AoYW8UBpVQWSZrZJ0fNbHTHW8,8364
 gooddata_eval/core/agentic/guardrail.py,sha256=qE9vADYilDnmqjcwEE-HGF3XsOcGjdvsKFfLSX9IFYQ,8251
-gooddata_eval/core/agentic/kda_skill.py,sha256=836jOEzvXjv_F-xurm3sUp0QaFPOqu3jZqy1kMSNLxI,17951
-gooddata_eval/core/agentic/metric_skill.py,sha256=zZs8cRcFdDLIV_rY6wQKd9o-0VbQYK6EOL8KbmZpk4A,14117
+gooddata_eval/core/agentic/kda_skill.py,sha256=XsKPhhEHQGJTPC3AufOgantxDTdFUwpjLpqlMfPLLEk,17974
+gooddata_eval/core/agentic/metric_skill.py,sha256=SeFJFPZLzWBLA2UxlH9q8kUDB6Ztb5ylxOApvd2lkCA,14831
 gooddata_eval/core/agentic/search_tool.py,sha256=L6yEs0hJymC_beYmily9ivw3M6tjLgDnF9_6kr7g11s,8020
-gooddata_eval/core/agentic/visualization.py,sha256=rZPN6CF_XpMsmPHSwzUP4iisLBL_99g2UIjIww_GbbQ,17129
+gooddata_eval/core/agentic/visualization.py,sha256=qhwJ06_SauPjOl1yJ3teZmf42YQjpcSwTy3yuc3kz30,17127
 gooddata_eval/core/chat/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/chat/sse_client.py,sha256=oYLxMWy6Hy0yoC8InnTkKLWN4Rsg1gSSIiE0FzaVa4o,16062
 gooddata_eval/core/dataset/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
@@ -45,8 +45,8 @@ gooddata_eval/core/reporting/console.py,sha256=kCHt4sCSaSRh0ZWqaSEJOl6_Jgb6uiouS
 gooddata_eval/core/reporting/json_report.py,sha256=sKPG9-uxzDsf6EJyBXoaHVIfN1g-E2rZ8fA0wtyBYb4,2943
 gooddata_eval/core/summary/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/summary/http_client.py,sha256=dPArJ6zoCu1W9xmYXFA1WMDNsSOkhn3gaCpYgCUp3gk,2174
-gooddata_eval-1.72.1.dev4.dist-info/METADATA,sha256=H_8L61m6K2yL9IR9kQPFcQKr001D6-j4Hd1Fz2kSkuc,9995
-gooddata_eval-1.72.1.dev4.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-gooddata_eval-1.72.1.dev4.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
-gooddata_eval-1.72.1.dev4.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
-gooddata_eval-1.72.1.dev4.dist-info/RECORD,,
+gooddata_eval-1.72.1.dev6.dist-info/METADATA,sha256=b3CWA-37bAb2hXjVrQJ3LiBgj0e5mla8iLp8Ga-ZNLk,9995
+gooddata_eval-1.72.1.dev6.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+gooddata_eval-1.72.1.dev6.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
+gooddata_eval-1.72.1.dev6.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
+gooddata_eval-1.72.1.dev6.dist-info/RECORD,,