gooddata-eval 1.68.1.dev3__tar.gz → 1.69.0__tar.gz

{gooddata_eval-1.68.1.dev3 → gooddata_eval-1.69.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-eval
-Version: 1.68.1.dev3
+Version: 1.69.0
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.68.1.dev3
+Requires-Dist: gooddata-sdk~=1.69.0
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.68.1.dev3 → gooddata_eval-1.69.0}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.68.1.dev3"
+version = "1.69.0"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.68.1.dev3",
+    "gooddata-sdk~=1.69.0",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",

{gooddata_eval-1.68.1.dev3 → gooddata_eval-1.69.0}/src/gooddata_eval/cli/main.py

@@ -3,6 +3,7 @@
 
 import argparse
 import sys
+import threading
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -139,14 +140,24 @@ def _parse_model_arg(val: str) -> tuple[str | None, str]:
 
 
 def _make_progress_callbacks(console: Console):
-    """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress."""
+    """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress.
+
+    A threading lock guards all console.print() calls so that concurrent
+    ``--concurrency 2+`` workers do not deadlock when stdout is piped
+    (e.g. running in a background process).
+    """
+    _print_lock = threading.Lock()
 
     def on_item_start(index: int, total: int, item: DatasetItem) -> None:
-        console.print(f"[dim]\\[{index}/{total}][/dim] [cyan]{item.id}[/cyan] {_truncate(item.question)}")
+        with _print_lock:
+            console.print(f"[dim]\\[{index}/{total}][/dim] [cyan]{item.id}[/cyan] {_truncate(item.question)}")
 
     def on_run_done(index: int, total: int, run_index: int, runs: int, passed: bool, latency: float) -> None:
         tag = "[green]pass[/green]" if passed else "[red]fail[/red]"
-        console.print(f"[dim]\\[{index}/{total}][/dim]     run {run_index}/{runs}  {tag}  [dim]{latency:.2f}s[/dim]")
+        with _print_lock:
+            console.print(
+                f"[dim]\\[{index}/{total}][/dim]     run {run_index}/{runs}  {tag}  [dim]{latency:.2f}s[/dim]"
+            )
 
     def on_item_done(index: int, total: int, report: ItemReport) -> None:
         if report.skipped:
@@ -165,7 +176,8 @@ def _make_progress_callbacks(console: Console):
                 f" [dim]({report.latency_s:.2f}s total, {report.avg_latency_s:.2f}s avg, "
                 f"quality={quality_str}, {report.runs} run(s))[/dim]"
             )
-        console.print(f"[dim]\\[{index}/{total}][/dim]   -> {tag} [cyan]{report.id}[/cyan]{suffix}")
+        with _print_lock:
+            console.print(f"[dim]\\[{index}/{total}][/dim]   -> {tag} [cyan]{report.id}[/cyan]{suffix}")
 
     return on_item_start, on_run_done, on_item_done
 

{gooddata_eval-1.68.1.dev3 → gooddata_eval-1.69.0}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -77,6 +77,8 @@ def _check_threshold(expected: CatalogMetricAlert, actual_args: dict) -> bool:
 
 
 def _check_trigger(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+    if expected.operator == "ANOMALY":
+        return True
     exp_trigger = expected.trigger
     act_trigger = actual_args.get("trigger", actual_args.get("triggerMode", "ALWAYS"))
     if exp_trigger in _ALWAYS_TRIGGER_VALUES:
@@ -235,27 +237,42 @@ class AgenticAlertSummary:
     best: AlertRunResult
 
 
+def _case_insensitive_get(d: dict, *keys: str) -> Any:
+    """Look up a value by key, preferring an exact match then a case-insensitive one."""
+    for k in keys:
+        if k in d:
+            return d[k]
+    lowered = {str(k).lower(): v for k, v in d.items()}
+    for k in keys:
+        if k.lower() in lowered:
+            return lowered[k.lower()]
+    return None
+
+
 def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
     """Parse expected_output dict into CatalogMetricAlert, accepting display-format or internal-format keys."""
-    operator = expected.get("operator") or expected.get("Operator") or "GREATER_THAN"
-    threshold = expected.get("threshold") or expected.get("Threshold")
-    threshold_from = expected.get("threshold_from")
-    threshold_to = expected.get("threshold_to")
-    trigger = expected.get("trigger") or expected.get("Trigger") or "not specified"
-
-    metric_id = expected.get("metric_id")
-    if not metric_id and "Metric" in expected:
-        m = re.search(r"\(([^)]+)\)\s*$", str(expected["Metric"]))
-        if m:
-            metric_id = m.group(1).strip()
-
-    raw_recip = expected.get("recipients") or expected.get("Recipient(s)") or []
+    operator = _case_insensitive_get(expected, "operator") or "GREATER_THAN"
+    threshold = _case_insensitive_get(expected, "threshold")
+    threshold_from = _case_insensitive_get(expected, "threshold_from")
+    threshold_to = _case_insensitive_get(expected, "threshold_to")
+
+    trigger = _case_insensitive_get(expected, "trigger") or "not specified"
+    trigger = _TRIGGER_DISPLAY_TO_API.get(trigger, trigger)
+
+    metric_id = _case_insensitive_get(expected, "metric_id")
+    if not metric_id:
+        metric_disp = _case_insensitive_get(expected, "metric")
+        if metric_disp:
+            m = re.search(r"\(([^)]+)\)\s*$", str(metric_disp))
+            metric_id = m.group(1).strip() if m else None
+
+    raw_recip = _case_insensitive_get(expected, "recipients", "recipient(s)") or []
     if isinstance(raw_recip, str):
         recipients = [r.strip() for r in raw_recip.replace(";", ",").split(",") if r.strip()]
     else:
         recipients = list(raw_recip)
 
-    filters = expected.get("filters") or expected.get("Time window/Filters")
+    filters = _case_insensitive_get(expected, "filters")
     if isinstance(filters, str) and any(kw in filters for kw in ("None", "All time")):
         filters = None
 

{gooddata_eval-1.68.1.dev3 → gooddata_eval-1.69.0}/src/gooddata_eval/core/agentic/conversation.py

@@ -102,7 +102,7 @@ def _activated_skills(tool_call_events: list[ToolCallEvent]) -> list[str]:
         if tc.function_name != "set_skills":
             continue
         args = tc.parsed_arguments() or {}
-        skills.extend(args.get("skills", []))
+        skills.extend(args.get("skill_names") or args.get("skills") or [])
     return list(set(skills))
 
 

{gooddata_eval-1.68.1.dev3 → gooddata_eval-1.69.0}/src/gooddata_eval/core/scoring.py

@@ -71,7 +71,9 @@ def validate_cross_references(viz: CreatedVisualization) -> tuple[bool, list[str
             continue
         using_val = filter_dict.get("using", "")
         using_uri = _resolve_alias_to_uri(using_val, fields)
-        if not using_uri.startswith(("metric/", "fact/")):
+        field_def = fields.get(using_val)
+        is_adhoc_agg = isinstance(field_def, AacQueryField) and bool(field_def.aggregation)
+        if not using_uri.startswith(("metric/", "fact/")) and not is_adhoc_agg:
             errors.append(
                 f"ranking filter '{filter_key}': using='{using_val}' "
                 f"resolves to '{using_uri}' — expected a metric/ or fact/ URI"

{gooddata_eval-1.68.1.dev3 → gooddata_eval-1.69.0}/tests/test_cli.py

@@ -530,3 +530,44 @@ def test_cli_rejects_negative_concurrency(monkeypatch, fixtures_dir):
         ]
     )
     assert exit_code == 2
+
+
+def test_progress_callbacks_thread_safe():
+    """Verify progress callbacks can be called from multiple threads without error."""
+    import io
+    import threading
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+
+    console = Console(file=io.StringIO(), force_terminal=False)
+    on_item_start, on_run_done, on_item_done = cli_main._make_progress_callbacks(console)
+
+    errors: list[Exception] = []
+
+    def _worker(index: int) -> None:
+        try:
+            item = DatasetItem(
+                id=f"test-{index}",
+                dataset_name="test",
+                test_kind="general_question",
+                question=f"Question {index}",
+                expected_output="answer",
+            )
+            on_item_start(index, 100, item)
+            on_run_done(index, 100, 1, 1, index % 2 == 0, 1.5)
+            report = ItemReport(id=f"test-{index}", dataset_name="test", test_kind="general_question")
+            report.runs = 1
+            report.latency_s = 1.5
+            report.pass_at_k = index % 2 == 0
+            on_item_done(index, 100, report)
+        except Exception as e:
+            errors.append(e)
+
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        futures = [pool.submit(_worker, i) for i in range(50)]
+        for f in as_completed(futures):
+            f.result()  # re-raise if any thread failed
+
+    assert not errors, f"Thread-safety violation: {errors}"
+    output = console.file.getvalue()
+    assert "test-1" in output
+    assert "test-49" in output