gooddata-eval 1.68.0__py3-none-any.whl

gooddata_eval/__init__.py

@@ -0,0 +1,6 @@
+# (C) 2026 GoodData Corporation
+"""gooddata-eval: evaluate the GoodData AI agent against your own datasets."""
+
+from gooddata_eval._version import __version__
+
+__all__ = ["__version__"]

gooddata_eval/_version.py

@@ -0,0 +1,7 @@
+# (C) 2026 GoodData Corporation
+from importlib import metadata
+
+try:
+    __version__ = metadata.version("gooddata-eval")
+except metadata.PackageNotFoundError:
+    __version__ = "unknown-version"

gooddata_eval/cli/__init__.py

@@ -0,0 +1 @@
+# (C) 2026 GoodData Corporation

gooddata_eval/cli/main.py

@@ -0,0 +1,382 @@
+# (C) 2026 GoodData Corporation
+"""`gd-eval` command-line entry point."""
+
+import argparse
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import httpx
+from gooddata_api_client.exceptions import ApiException
+from rich.console import Console
+from rich.table import Table
+
+from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import RunConfig
+from gooddata_eval.core.connection import ConnectionError_, resolve_connection
+from gooddata_eval.core.dataset.local import load_local_dataset
+from gooddata_eval.core.langfuse.sink import LangfuseSink
+from gooddata_eval.core.models import ChatResult, DatasetItem
+from gooddata_eval.core.reporting.console import render_comparison, render_console
+from gooddata_eval.core.reporting.json_report import write_multi_model_report
+from gooddata_eval.core.runner import ItemReport, run_items
+from gooddata_eval.core.summary.http_client import SummaryClient
+from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController
+
+_EXIT_OK = 0
+_EXIT_OPERATIONAL_ERROR = 2
+_SUMMARY_TEST_KIND = "dashboard_summary"
+
+
+class _RoutingBackend:
+    """Dispatch each item to the right backend by test_kind.
+
+    `dashboard_summary` items go to the dedicated summary endpoint; everything
+    else uses the conversational chat endpoint.
+    """
+
+    def __init__(self, chat: ChatClient, summary: SummaryClient):
+        self._chat = chat
+        self._summary = summary
+
+    def ask(self, item: DatasetItem) -> ChatResult:
+        if item.test_kind == _SUMMARY_TEST_KIND:
+            return self._summary.ask(item)
+        return self._chat.ask(item)
+
+    def close(self) -> None:
+        for backend in (self._chat, self._summary):
+            if hasattr(backend, "close"):
+                backend.close()
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="gd-eval", description="Evaluate the GoodData AI agent.")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    run = sub.add_parser("run", help="Run an evaluation dataset.")
+    run.add_argument("--host", help="GoodData host URL.")
+    run.add_argument("--token", help="API token (or set GOODDATA_TOKEN).")
+    run.add_argument("--profile", help="Profile name in ~/.gooddata/profiles.yaml.")
+    run.add_argument("--workspace", required=True, help="Workspace id.")
+    source = run.add_mutually_exclusive_group(required=True)
+    source.add_argument("--dataset", help="Path to a folder of dataset JSON files.")
+    source.add_argument("--langfuse-dataset", dest="langfuse_dataset", help="Langfuse dataset name.")
+    run.add_argument(
+        "--model",
+        action="append",
+        dest="models",
+        metavar="MODEL",
+        help=(
+            "Model id to evaluate (e.g. --model gpt-5.2). "
+            "Prefix with provider name or id to disambiguate: "
+            "--model ProviderName/gpt-5.2 or --model provider_id/gpt-5.2. "
+            "Repeat to compare multiple models. "
+            "Default: workspace's current active model."
+        ),
+    )
+    run.add_argument("--runs", type=int, default=2, help="Independent runs per item (pass@K). Default 2.")
+    run.add_argument(
+        "--concurrency",
+        type=int,
+        default=1,
+        help="Number of items evaluated concurrently (default 1 = sequential). "
+        "Increase to load-test the agent under simultaneous requests.",
+    )
+    run.add_argument("--json", dest="json_path", help="Write a JSON report to this path.")
+    run.add_argument("--quiet", action="store_true", help="Suppress per-item progress output.")
+    run.add_argument(
+        "--langfuse",
+        action="store_true",
+        help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).",
+    )
+    models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.")
+    models_cmd.add_argument("--host", help="GoodData host URL.")
+    models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).")
+    models_cmd.add_argument("--profile", help="Profile name in ~/.gooddata/profiles.yaml.")
+    models_cmd.add_argument(
+        "--workspace",
+        help="Workspace id. When provided, marks the currently active model.",
+    )
+    return parser
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    return _build_parser().parse_args(argv)
+
+
+def _truncate(text: str, limit: int = 80) -> str:
+    return text if len(text) <= limit else text[: limit - 1] + "…"
+
+
+def _parse_model_arg(val: str) -> tuple[str | None, str]:
+    """Parse a model argument that may include a provider prefix.
+
+    Accepts two forms:
+      "gpt-5.2"                   → (None, "gpt-5.2")
+      "ProviderName/gpt-5.2"      → ("ProviderName", "gpt-5.2")
+      "provider_id.../model_id"   → ("provider_id...", "model_id")
+
+    The provider part (if present) is passed to resolve_and_activate and
+    accepted as either a provider name or provider id.
+    """
+    if "/" in val:
+        provider_ref, _, model_id = val.partition("/")
+        return provider_ref.strip() or None, model_id.strip()
+    return None, val
+
+
+def _make_progress_callbacks(console: Console):
+    """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress."""
+
+    def on_item_start(index: int, total: int, item: DatasetItem) -> None:
+        console.print(f"[dim]\\[{index}/{total}][/dim] [cyan]{item.id}[/cyan] {_truncate(item.question)}")
+
+    def on_run_done(index: int, total: int, run_index: int, runs: int, passed: bool, latency: float) -> None:
+        tag = "[green]pass[/green]" if passed else "[red]fail[/red]"
+        console.print(f"[dim]\\[{index}/{total}][/dim]     run {run_index}/{runs}  {tag}  [dim]{latency:.2f}s[/dim]")
+
+    def on_item_done(index: int, total: int, report: ItemReport) -> None:
+        if report.skipped:
+            tag = "[yellow]SKIP[/yellow]"
+        elif report.error:
+            tag = "[red]ERR [/red]"
+        elif report.pass_at_k:
+            tag = "[green]PASS[/green]"
+        else:
+            tag = "[red]FAIL[/red]"
+        if report.skipped:
+            suffix = ""
+        else:
+            quality_str = f"{report.quality_score:.0%}"
+            suffix = (
+                f" [dim]({report.latency_s:.2f}s total, {report.avg_latency_s:.2f}s avg, "
+                f"quality={quality_str}, {report.runs} run(s))[/dim]"
+            )
+        console.print(f"[dim]\\[{index}/{total}][/dim]   -> {tag} [cyan]{report.id}[/cyan]{suffix}")
+
+    return on_item_start, on_run_done, on_item_done
+
+
+def _load_dataset(config: RunConfig):
+    if config.dataset_folder is not None:
+        return load_local_dataset(config.dataset_folder)
+    from gooddata_eval.core.dataset.langfuse_source import load_langfuse_dataset  # noqa: PLC0415
+
+    if config.langfuse_dataset is None:  # pragma: no cover - argparse mutually-exclusive group guarantees one is set
+        raise ValueError("Either --dataset or --langfuse-dataset is required.")
+    return load_langfuse_dataset(config.langfuse_dataset)
+
+
+def _list_models(host: str, token: str, workspace_id: str | None) -> int:
+    """List all LLM providers and their models; mark the active one if --workspace given."""
+    from gooddata_eval.core.workspace import WorkspaceModelController  # noqa: PLC0415
+
+    controller = WorkspaceModelController(host, token, workspace_id or "")
+    info = controller._provider_info()  # {provider_id: {name, models: [{id, family}]}}
+
+    active_provider_id: str | None = None
+    active_model_id: str | None = None
+    if workspace_id:
+        active = controller.get_active()
+        if active:
+            active_provider_id = active.provider_id
+            active_model_id = active.default_model_id
+
+    console = Console()
+
+    if not info:
+        console.print("[yellow]No LLM providers configured in this organisation.[/yellow]")
+        return _EXIT_OK
+
+    table = Table(title=f"LLM Providers and Models{f' (workspace: {workspace_id})' if workspace_id else ''}")
+    table.add_column("Provider")
+    table.add_column("Provider ID")
+    table.add_column("Model ID")
+    table.add_column("Family")
+    table.add_column("Active")
+
+    for provider_id, pinfo in sorted(info.items(), key=lambda kv: kv[1].get("name") or kv[0]):
+        name = pinfo.get("name") or provider_id
+        models = pinfo.get("models") or []
+        if not models:
+            is_active_provider = provider_id == active_provider_id
+            table.add_row(name, provider_id, "[dim](none listed)[/dim]", "", "◀" if is_active_provider else "")
+        for i, model in enumerate(models):
+            model_id = model.get("id", "?") if isinstance(model, dict) else str(model)
+            family = model.get("family", "") if isinstance(model, dict) else ""
+            is_active = provider_id == active_provider_id and model_id == active_model_id
+            active_marker = "[green]◀ active[/green]" if is_active else ""
+            table.add_row(
+                name if i == 0 else "",
+                provider_id if i == 0 else "",
+                model_id,
+                family,
+                active_marker,
+            )
+
+    console.print(table)
+    return _EXIT_OK
+
+
+def _run(config: RunConfig) -> int:
+    if config.log_to_langfuse and config.langfuse_dataset is None:
+        print(
+            "error: --langfuse requires --langfuse-dataset (local datasets have no Langfuse item ids to link to).",
+            file=sys.stderr,
+        )
+        return _EXIT_OPERATIONAL_ERROR
+
+    items = _load_dataset(config)
+    models = config.models or []
+    run_ts = datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M")
+    n_models = len(models) if models else 1
+
+    controller = WorkspaceModelController(config.host, config.token, config.workspace_id)
+    original_active = controller.get_active()
+
+    progress_console = Console(stderr=True) if not config.quiet else None
+    if progress_console:
+        multi_suffix = f" — {n_models} model(s)" if n_models > 1 else ""
+        progress_console.print(f"Evaluating {len(items)} item(s) on workspace '{config.workspace_id}'{multi_suffix}")
+
+    reports: list = []
+    try:
+        for k, model_id in enumerate(models or [None], start=1):
+            provider_ref, bare_model_id = _parse_model_arg(model_id or "")
+            effective_provider = provider_ref
+            try:
+                resolved = controller.resolve_and_activate(bare_model_id or None, effective_provider)
+            except (ModelResolutionError, httpx.HTTPError, ApiException, RuntimeError) as exc:
+                print(f"warning: skipping model '{model_id}': {exc}", file=sys.stderr)
+                continue
+
+            if progress_console:
+                if n_models > 1:
+                    progress_console.print(f"\n── Model {k}/{n_models}: {resolved.model_id} " + "─" * 40)
+                else:
+                    switched = " [switched active provider]" if resolved.switched else ""
+                    provider_display = resolved.provider_name or resolved.provider_id
+                    progress_console.print(f"Provider={provider_display}, model={resolved.model_id}{switched}")
+
+            run_name = f"gd-eval-{run_ts}-{resolved.model_id}"
+            if progress_console and config.log_to_langfuse:
+                progress_console.print(f"Logging to Langfuse run '{run_name}'...")
+
+            on_item_start, on_run_done, on_item_done = (None, None, None)
+            if progress_console:
+                on_item_start, on_run_done, on_item_done = _make_progress_callbacks(progress_console)
+
+            on_langfuse_item_done = None
+            if config.log_to_langfuse:
+                assert config.langfuse_dataset is not None
+                sink = LangfuseSink(
+                    dataset_name=config.langfuse_dataset,
+                    run_name=run_name,
+                    model_id=resolved.model_id,
+                    provider_type=resolved.provider_type,
+                )
+
+                def on_langfuse_item_done(
+                    index: int,
+                    total: int,
+                    report: ItemReport,
+                    _sink: LangfuseSink = sink,
+                    _model_id: str = resolved.model_id,
+                    _provider_type: str = resolved.provider_type,
+                ) -> None:
+                    _sink.log_item(report, dataset_item_id=report.id)
+
+            backend = _RoutingBackend(
+                ChatClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
+                SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
+            )
+            try:
+                report = run_items(
+                    items,
+                    backend,
+                    runs=config.runs,
+                    model=resolved.model_id,
+                    provider_name=resolved.provider_name or resolved.provider_id,
+                    provider_type=resolved.provider_type,
+                    workspace_id=config.workspace_id,
+                    on_item_start=on_item_start,
+                    on_run_done=on_run_done,
+                    on_item_done=on_item_done,
+                    on_langfuse_item_done=on_langfuse_item_done,
+                    concurrency=config.concurrency,
+                )
+            finally:
+                if hasattr(backend, "close"):
+                    backend.close()
+
+            skipped_kinds = sorted({i.test_kind for i in report.items if i.skipped})
+            if skipped_kinds:
+                print(
+                    f"warning: skipped {sum(i.skipped for i in report.items)} item(s) with "
+                    f"unsupported test_kind(s): {', '.join(skipped_kinds)}",
+                    file=sys.stderr,
+                )
+
+            render_console(report)
+            reports.append(report)
+    finally:
+        try:
+            controller.restore(original_active)
+        except Exception as _restore_exc:
+            print(
+                f"warning: failed to restore workspace active model: {_restore_exc}",
+                file=sys.stderr,
+            )
+
+    if not reports:
+        print("error: no models evaluated successfully.", file=sys.stderr)
+        return _EXIT_OPERATIONAL_ERROR
+
+    if len(reports) > 1:
+        render_comparison(reports)
+
+    if config.json_path is not None:
+        write_multi_model_report(reports, config.json_path)
+
+    return _EXIT_OK
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv if argv is not None else sys.argv[1:])
+    if hasattr(args, "concurrency") and args.concurrency < 1:
+        print("error: --concurrency must be >= 1.", file=sys.stderr)
+        return _EXIT_OPERATIONAL_ERROR
+    try:
+        host, token = resolve_connection(host=args.host, token=args.token, profile=args.profile)
+        if args.command == "models":
+            return _list_models(host, token, getattr(args, "workspace", None))
+        config = RunConfig(
+            host=host,
+            token=token,
+            workspace_id=args.workspace,
+            dataset_folder=Path(args.dataset) if args.dataset else None,
+            langfuse_dataset=args.langfuse_dataset,
+            models=args.models or [],
+            runs=args.runs,
+            concurrency=args.concurrency,
+            json_path=Path(args.json_path) if args.json_path else None,
+            log_to_langfuse=args.langfuse,
+            quiet=args.quiet,
+        )
+        return _run(config)
+    except (
+        ConnectionError_,
+        ModelResolutionError,
+        FileNotFoundError,
+        ValueError,
+        httpx.HTTPError,
+        ApiException,
+        RuntimeError,
+    ) as e:
+        print(f"error: {e}", file=sys.stderr)
+        return _EXIT_OPERATIONAL_ERROR
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

gooddata_eval/core/__init__.py

@@ -0,0 +1 @@
+# (C) 2026 GoodData Corporation

gooddata_eval/core/chat/__init__.py

@@ -0,0 +1 @@
+# (C) 2026 GoodData Corporation

gooddata_eval/core/chat/sse_client.py

@@ -0,0 +1,181 @@
+# (C) 2026 GoodData Corporation
+"""SSE chat client for the agentic AI conversations API.
+
+Ported from gdc-nas tavern-e2e app/sse_client.py (httpx instead of requests).
+
+Why not gooddata_sdk.compute.ai_chat / ai_chat_stream? Those target the legacy
+``/api/v1/actions/workspaces/{ws}/ai/chat[Stream]`` endpoint and expose a different
+visualization shape (``metrics``/``dimensionality``). This evaluator scores the
+*agentic* visualization (AAC ``query.fields`` shape) returned by the newer
+``/api/v1/ai/workspaces/{ws}/chat/conversations`` endpoint, which is not yet
+present in the generated api-client. When that endpoint lands in the SDK, this
+module is the single place to swap — the runner only depends on the ChatBackend
+protocol, not on this class.
+"""
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, Iterable
+
+import httpx
+
+from gooddata_eval.core.models import ChatResult, DatasetItem
+
+SSE_DATA_PREFIX = "data: "
+
+
+@dataclass
+class _SseAccumulator:
+    text_parts: list[str] = field(default_factory=list)
+    viz_reasoning_parts: list[str] = field(default_factory=list)
+    visualizations: list[dict[str, Any]] = field(default_factory=list)
+    tool_call_events: list[dict[str, Any]] = field(default_factory=list)
+    call_id_to_event_index: dict[str, int] = field(default_factory=dict)
+    reasoning_steps: list[dict[str, Any]] = field(default_factory=list)
+    adhoc_viz_args: list[dict[str, Any]] = field(default_factory=list)
+
+
+def _handle_text(content: dict[str, Any], acc: _SseAccumulator) -> None:
+    text = content.get("text", "")
+    if text:
+        acc.text_parts.append(text)
+
+
+def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator) -> None:
+    for part in content.get("parts", []):
+        ptype = part.get("type")
+        if ptype == "text":
+            t = part.get("text", "")
+            if t:
+                acc.text_parts.append(t)
+                acc.viz_reasoning_parts.append(t)
+        elif ptype == "visualization" and part.get("visualization"):
+            acc.visualizations.append(part["visualization"])
+
+
+def _handle_reasoning(content: dict[str, Any], acc: _SseAccumulator) -> None:
+    summary = content.get("summary", "")
+    if summary:
+        acc.reasoning_steps.append({"summary": summary})
+
+
+def _handle_tool_call(content: dict[str, Any], acc: _SseAccumulator) -> None:
+    call_id = content.get("callId", "")
+    acc.call_id_to_event_index[call_id] = len(acc.tool_call_events)
+    acc.tool_call_events.append(
+        {
+            "functionName": content.get("name", ""),
+            "functionArguments": json.dumps(content.get("arguments", {})),
+            "result": None,
+        }
+    )
+    # Stash visualization definition from create_adhoc_visualization so we can
+    # evaluate the agent's intended answer even when the data source call fails.
+    if content.get("name") == "create_adhoc_visualization":
+        viz = (content.get("arguments") or {}).get("visualization")
+        if viz and isinstance(viz, dict):
+            acc.adhoc_viz_args.append(viz)
+
+
+def _handle_tool_result(content: dict[str, Any], acc: _SseAccumulator) -> None:
+    call_id = content.get("callId", "")
+    idx = acc.call_id_to_event_index.get(call_id)
+    if idx is not None:
+        acc.tool_call_events[idx]["result"] = content.get("result", "")
+
+
+def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
+    payload: dict[str, Any] = {
+        "textResponse": "\n".join(acc.text_parts) or None,
+        "toolCallEvents": acc.tool_call_events,
+    }
+    if acc.visualizations:
+        payload["createdVisualizations"] = {
+            "objects": acc.visualizations,
+            "reasoning": "\n".join(acc.viz_reasoning_parts),
+        }
+    elif acc.adhoc_viz_args:
+        # Fallback: the agent produced a correct visualization definition via
+        # create_adhoc_visualization but the call failed (e.g. data source not
+        # accessible). The last attempt is the agent's best answer.
+        payload["createdVisualizations"] = {
+            "objects": [acc.adhoc_viz_args[-1]],
+            "reasoning": "\n".join(acc.viz_reasoning_parts),
+        }
+    return ChatResult.model_validate(payload)
+
+
+def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
+    """Parse an SSE stream (iterable of decoded lines) into a ChatResult."""
+    acc = _SseAccumulator()
+    for raw_line in lines:
+        line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else raw_line
+        if not line or line.startswith("event: ") or not line.startswith(SSE_DATA_PREFIX):
+            continue
+        data_str = line[len(SSE_DATA_PREFIX) :]
+        try:
+            event_data = json.loads(data_str)
+        except json.JSONDecodeError:
+            continue
+        if "statusCode" in event_data:
+            raise RuntimeError(f"SSE error {event_data.get('statusCode')}: {event_data.get('detail')}")
+        item = event_data.get("item")
+        if not item:
+            continue
+        role = item.get("role")
+        content: dict[str, Any] = item.get("content") or {}
+        ctype = content.get("type")
+        if role == "assistant":
+            if ctype == "text":
+                _handle_text(content, acc)
+            elif ctype == "multipart":
+                _handle_multipart(content, acc)
+            elif ctype == "reasoning":
+                _handle_reasoning(content, acc)
+            elif ctype == "toolCall":
+                _handle_tool_call(content, acc)
+        elif role == "tool" and ctype == "toolResult":
+            _handle_tool_result(content, acc)
+    return _build_chat_result(acc)
+
+
+class ChatClient:
+    """Single-turn AI chat client over the GoodData AI conversation endpoints."""
+
+    def __init__(self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0):
+        self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
+        self._auth = {"Authorization": f"Bearer {token}"}
+        self._client = httpx.Client(timeout=timeout)
+
+    def _create_conversation(self) -> str:
+        resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"})
+        resp.raise_for_status()
+        body = resp.json()
+        if "conversationId" not in body:
+            raise ValueError(f"GoodData /chat/conversations response missing 'conversationId': {body}")
+        return body["conversationId"]
+
+    def _delete_conversation(self, conversation_id: str) -> None:
+        try:
+            self._client.delete(f"{self._base}/{conversation_id}", headers=self._auth)
+        except httpx.HTTPError:
+            pass  # best-effort cleanup
+
+    def _send_message(self, conversation_id: str, question: str) -> ChatResult:
+        url = f"{self._base}/{conversation_id}/messages"
+        headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json"}
+        body = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
+        with self._client.stream("POST", url, json=body, headers=headers) as resp:
+            resp.raise_for_status()
+            return parse_sse_lines(resp.iter_lines())
+
+    def ask(self, item: DatasetItem) -> ChatResult:
+        """Run one single-turn conversation: create, send, parse, clean up."""
+        conversation_id = self._create_conversation()
+        try:
+            return self._send_message(conversation_id, item.question)
+        finally:
+            self._delete_conversation(conversation_id)
+
+    def close(self) -> None:
+        self._client.close()

gooddata_eval/core/config.py

@@ -0,0 +1,20 @@
+# (C) 2026 GoodData Corporation
+"""Validated run configuration produced by the CLI and consumed by the runner."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class RunConfig:
+    host: str
+    token: str
+    workspace_id: str
+    dataset_folder: Path | None = None
+    langfuse_dataset: str | None = None
+    models: list[str] = field(default_factory=list)
+    runs: int = 2
+    concurrency: int = 1
+    json_path: Path | None = None
+    log_to_langfuse: bool = False
+    quiet: bool = False

gooddata_eval/core/connection.py

@@ -0,0 +1,33 @@
+# (C) 2026 GoodData Corporation
+"""Resolve (host, token) from explicit flags, environment, or a gooddata.yaml profile."""
+
+import os
+
+from gooddata_sdk.utils import profile_content
+
+
+class ConnectionError_(Exception):
+    """Raised when host/token cannot be resolved."""
+
+
+def resolve_connection(host: str | None, token: str | None, profile: str | None) -> tuple[str, str]:
+    """Resolve connection parameters.
+
+    Precedence: explicit flags > GOODDATA_TOKEN env (token only) > profile file.
+
+    Raises:
+        ConnectionError_: host or token could not be determined.
+    """
+    resolved_host = host
+    resolved_token = token or os.environ.get("GOODDATA_TOKEN")
+
+    if profile is not None and (resolved_host is None or resolved_token is None):
+        content = profile_content(profile)
+        resolved_host = resolved_host or content.get("host")
+        resolved_token = resolved_token or content.get("token")
+
+    if not resolved_host:
+        raise ConnectionError_("Missing host. Pass --host or use a --profile that defines it.")
+    if not resolved_token:
+        raise ConnectionError_("Missing token. Pass --token, set GOODDATA_TOKEN, or use a --profile that defines it.")
+    return resolved_host, resolved_token

gooddata_eval/core/dataset/__init__.py

@@ -0,0 +1 @@
+# (C) 2026 GoodData Corporation