gooddata-eval 1.76.1.dev4__py3-none-any.whl → 1.77.1.dev1__py3-none-any.whl

gooddata_eval/cli/agentic_runner.py

@@ -15,11 +15,11 @@ from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
 from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection
 from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
 from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill
+from gooddata_eval.core.agentic.document_skill import evaluate_agentic_document_skill
 from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
 from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
 from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill
 from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
-from gooddata_eval.core.agentic.report_skill import evaluate_agentic_report_skill
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
@@ -38,6 +38,7 @@ class _LfKw(TypedDict, total=False):
     model_version_override: str | None
     reasoning_effort: ReasoningEffort | None
     submit_trace_link: SubmitTraceLink
+    run_metadata_extra: dict[str, Any] | None
 
 
 AGENTIC_TEST_KINDS = frozenset(
@@ -47,7 +48,7 @@ AGENTIC_TEST_KINDS = frozenset(
         "agentic_metric_skill",
         "agentic_alert_skill",
         "agentic_dashboard_skill",
-        "agentic_report_skill",
+        "agentic_document_skill",
         "agentic_search",
         "agentic_general_question",
         "agentic_guardrail",
@@ -98,8 +99,8 @@ PARALLEL_SAFE_TEST_KINDS = frozenset(
 #
 # agentic_dashboard_skill is absent by default rather than by evidence: gen-ai holds the draft and
 # any chart it authors in conversation state and writes neither until a user saves from the UI, so
-# it is a candidate for the allowlist once the dataset has runs behind it. agentic_report_skill is
-# absent for the same reason: the report draft stays in conversation state until a user saves it.
+# it is a candidate for the allowlist once the dataset has runs behind it. agentic_document_skill is
+# absent for the same reason: the document draft stays in conversation state until a user saves it.
 WORKSPACE_MUTATING_TEST_KINDS = frozenset(AGENTIC_TEST_KINDS) - PARALLEL_SAFE_TEST_KINDS
 
 
@@ -152,11 +153,13 @@ def _dispatch_agentic(
     agent_id: str | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
     gate: EvalGate = DEFAULT_GATE,
+    run_metadata_extra: dict[str, Any] | None = None,
 ) -> AgenticEvalOutcome:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind.
 
     `gate` reaches every kind except those in UNGATED_AGENTIC_TEST_KINDS, which have no K
     to gate over; the CLI refuses --gate power for a dataset containing one.
+    `run_metadata_extra` reaches every kind unchanged.
 
     Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps,
     conversation_id, response_id, detail) on success and attaches the same four attributes
@@ -172,6 +175,7 @@ def _dispatch_agentic(
         "model_version_override": model_version_override,
         "reasoning_effort": reasoning_effort,
         "submit_trace_link": submit_trace_link,
+        "run_metadata_extra": run_metadata_extra,
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
@@ -214,8 +218,8 @@ def _dispatch_agentic(
             user_context=item.user_context,
             **lf_kw,
         )
-    elif kind == "agentic_report_skill":
-        return evaluate_agentic_report_skill(
+    elif kind == "agentic_document_skill":
+        return evaluate_agentic_document_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -400,6 +404,7 @@ def run_agentic_items(
     agent_id: str | None = None,
     concurrency: int = 1,
     gate: EvalGate = DEFAULT_GATE,
+    tags: tuple[str, ...] = (),
 ) -> EvalReport:
     """Run agentic items through evaluate_agentic_* and return an EvalReport.
 
@@ -408,9 +413,14 @@ def run_agentic_items(
     from the parallel ones -- a metric being created and dropped mid-run would otherwise be
     visible to a catalog-reading item running alongside it.
 
+    ``tags`` are added to every Langfuse trace gd-eval writes for these items, after the
+    automatic ones. They reach Langfuse whenever trace linking runs, with or without
+    ``use_langfuse``.
+
     Results are collected in dataset order regardless of completion order.
     """
     langfuse = make_langfuse_client() if use_langfuse else None
+    run_metadata_extra: dict[str, Any] | None = {"tags": list(tags)} if tags else None
 
     report = EvalReport(model=model_version, gate=normalize_gate(gate))
     total = len(items)
@@ -452,6 +462,7 @@ def run_agentic_items(
                 agent_id,
                 submit_trace_link=linker.submit,
                 gate=gate,
+                run_metadata_extra=run_metadata_extra,
             )
             if isinstance(outcome, AgenticEvalOutcome):
                 reasoning_steps = outcome.reasoning_steps

gooddata_eval/cli/main.py

@@ -183,6 +183,17 @@ def _build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).",
     )
+    run.add_argument(
+        "--tag",
+        dest="tags",
+        action="append",
+        default=[],
+        type=_parse_tag,
+        metavar="TAG",
+        help="Custom tag for every Langfuse trace gd-eval writes, next to the automatic ones. "
+        "Repeat for several tags. Applied wherever traces are written: single-turn kinds only "
+        "with --langfuse, agentic kinds whenever LANGFUSE_* env vars are set.",
+    )
     run.add_argument(
         "--agent-id",
         dest="agent_id",
@@ -377,6 +388,14 @@ def _parse_model_arg(val: str) -> tuple[str | None, str]:
     return None, val
 
 
+def _parse_tag(val: str) -> str:
+    """Strip one --tag value and reject a blank one, which Langfuse would store as an empty tag."""
+    tag = val.strip()
+    if not tag:
+        raise argparse.ArgumentTypeError("tag must not be blank")
+    return tag
+
+
 def _make_progress_callbacks(console: Console):
     """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress.
 
@@ -549,6 +568,7 @@ def _run(config: RunConfig) -> int:
                     model_id=resolved.model_id,
                     provider_type=resolved.provider_type,
                     reasoning_effort=config.reasoning_effort,
+                    tags=config.tags,
                 )
 
                 def on_langfuse_item_done(
@@ -574,6 +594,7 @@ def _run(config: RunConfig) -> int:
                     model_version=resolved.model_id,
                     reasoning_effort=config.reasoning_effort,
                     use_langfuse=config.log_to_langfuse,
+                    tags=config.tags,
                     run_ts=run_ts,
                     on_item_start=on_item_start,
                     on_item_done=on_item_done,
@@ -728,6 +749,7 @@ def main(argv: list[str] | None = None) -> int:
             agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"),
             turn_timeout_s=args.turn_timeout,
             item_timeout_s=args.item_timeout,
+            tags=tuple(dict.fromkeys(args.tags)),
         )
         return _run(config)
     except (

gooddata_eval/core/agentic/__init__.py

@@ -24,6 +24,14 @@ from gooddata_eval.core.agentic.dashboard_skill import (
     evaluate_agentic_dashboard_skill,
     run_agentic_dashboard_skill,
 )
+from gooddata_eval.core.agentic.document_skill import (
+    AgenticDocumentSummary,
+    DocumentEvaluation,
+    DocumentRunResult,
+    DocumentSkillAssertionError,
+    evaluate_agentic_document_skill,
+    run_agentic_document_skill,
+)
 from gooddata_eval.core.agentic.general_question import (
     AgenticGeneralQuestionSummary,
     GeneralQuestionAssertionError,
@@ -53,14 +61,6 @@ from gooddata_eval.core.agentic.metric_skill import (
     evaluate_agentic_metric_skill,
     run_agentic_metric_skill,
 )
-from gooddata_eval.core.agentic.report_skill import (
-    AgenticReportSummary,
-    ReportEvaluation,
-    ReportRunResult,
-    ReportSkillAssertionError,
-    evaluate_agentic_report_skill,
-    run_agentic_report_skill,
-)
 from gooddata_eval.core.agentic.search_tool import (
     AgenticSearchSummary,
     SearchResult,
@@ -76,6 +76,14 @@ from gooddata_eval.core.agentic.visualization import (
     run_agentic_visualization,
 )
 
+# Deprecated names of the document skill, kept for existing callers.
+AgenticReportSummary = AgenticDocumentSummary
+ReportEvaluation = DocumentEvaluation
+ReportRunResult = DocumentRunResult
+ReportSkillAssertionError = DocumentSkillAssertionError
+evaluate_agentic_report_skill = evaluate_agentic_document_skill
+run_agentic_report_skill = run_agentic_document_skill
+
 __all__ = [
     "AgenticAlertSummary",
     "AgenticDashboardSummary",
@@ -83,6 +91,7 @@ __all__ = [
     "AgenticGuardrailSummary",
     "AgenticKdaSummary",
     "AgenticMetricSummary",
+    "AgenticDocumentSummary",
     "AgenticReportSummary",
     "AgenticSearchSummary",
     "AgenticRunSummary",
@@ -95,6 +104,9 @@ __all__ = [
     "DashboardEvaluation",
     "DashboardRunResult",
     "DashboardSkillAssertionError",
+    "DocumentEvaluation",
+    "DocumentRunResult",
+    "DocumentSkillAssertionError",
     "GeneralQuestionAssertionError",
     "GeneralQuestionResult",
     "GuardrailAssertionError",
@@ -116,6 +128,7 @@ __all__ = [
     "evaluate_agentic_alert_skill",
     "evaluate_agentic_conversation",
     "evaluate_agentic_dashboard_skill",
+    "evaluate_agentic_document_skill",
     "evaluate_agentic_general_question",
     "evaluate_agentic_guardrail",
     "evaluate_agentic_kda_skill",
@@ -126,6 +139,7 @@ __all__ = [
     "run_agentic_alert_skill",
     "run_agentic_conversation",
     "run_agentic_dashboard_skill",
+    "run_agentic_document_skill",
     "run_agentic_general_question",
     "run_agentic_guardrail",
     "run_agentic_kda_skill",

gooddata_eval/core/agentic/_langfuse.py

@@ -3,6 +3,8 @@
 
 from __future__ import annotations
 
+import contextvars
+import functools
 import logging
 import os
 import threading
@@ -10,7 +12,7 @@ import time
 from collections.abc import Iterator
 from contextlib import contextmanager
 from datetime import datetime, timedelta, timezone
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from gooddata_eval.core.agentic._trace_linker import link_cancel_event, linking_is_inline, warn_from_worker
 from gooddata_eval.core.config import ReasoningEffort, env_flag, normalize_reasoning_effort
@@ -21,12 +23,26 @@ from gooddata_eval.core.langfuse.experiment import (
     ExperimentRun,
     ScoreTarget,
     build_experiment_root_span,
+    trace_tags,
+)
+from gooddata_eval.core.langfuse.item_scope import (
+    SDK_EXPERIMENT_ENVIRONMENT,
+    ItemTraceScope,
+    JoinedRun,
+    joined_run,
+    joined_turns,
+    release_joined,
+    trace_ids_for,
 )
 
 # Part of this module's public surface: external callers import both names from here.
 from gooddata_eval.core.langfuse.observations import TraceSummary as _TraceObj  # noqa: F401
+from gooddata_eval.core.langfuse.observations import summarize_traces
 from gooddata_eval.core.langfuse.otlp import Span
 
+if TYPE_CHECKING:
+    from gooddata_eval.core.agentic._trace_linker import RunIdentity
+
 _log = logging.getLogger(__name__)
 
 
@@ -115,16 +131,22 @@ def get_model_version(
     if override:
         return override
     try:
-        from gooddata_sdk import GoodDataSdk  # noqa: PLC0415
-
-        sdk = GoodDataSdk.create(host, token)
-        setting = sdk.catalog_workspace.get_workspace_setting(workspace_id, "activeLlmProvider")
-        model = (setting.content or {}).get("defaultModelId") or None
-        if model:
-            return model
+        return _active_llm_model(host, token, workspace_id)
     except Exception:
-        pass
-    return ""
+        return ""
+
+
+@functools.cache
+def _active_llm_model(host: str, token: str, workspace_id: str) -> str:
+    """The workspace's default model id, read once per workspace; a failed read raises and is retried.
+
+    Every joined item resolves its run context before the first turn, on the item's own clock.
+    """
+    from gooddata_sdk import GoodDataSdk  # noqa: PLC0415
+
+    sdk = GoodDataSdk.create(host, token)
+    setting = sdk.catalog_workspace.get_workspace_setting(workspace_id, "activeLlmProvider")
+    return (setting.content or {}).get("defaultModelId") or ""
 
 
 def _matches_session(trace: Any, session_id: str) -> bool:
@@ -200,6 +222,22 @@ def _wait_between_attempts(delay: float) -> bool:
     return not cancel.is_set()
 
 
+def _fetch_joined_trace(langfuse: HttpxLangfuseClient, session_id: str, *_window: Any) -> list[Any]:
+    """The trace a joined conversation's gen-ai turns were put in, read by its id.
+
+    Empty until every joined turn's ``conversation.send_message`` row is ingested: a turn row
+    lands only when its turn ends, so an early read would cover the first turns alone.
+    ``joined_turns`` counts one per sent turn, but a turn the chat client retried after
+    gen-ai opened it leaves a row per attempt, so the count can be met while a later turn
+    is still in flight; the read then understates cost and latency.
+    """
+    trace_id = trace_ids_for(session_id)[0]
+    rows = langfuse.list_observations_for_trace(trace_id)
+    if sum(r.get("name") == "conversation.send_message" for r in rows) < joined_turns(session_id):
+        return []
+    return [t for t in summarize_traces(rows) if t.id == trace_id]
+
+
 def find_traces_per_conversation(
     langfuse: Any,
     conversation_ids: list[str],
@@ -245,12 +283,14 @@ def find_traces_per_conversation(
             break
         delay = _INITIAL_DELAY
         found: list[Any] = []
+        joined = joined_run(cid) is not None and isinstance(langfuse, HttpxLangfuseClient)
+        fetch = _fetch_joined_trace if joined else _fetch_traces_for_session
         for _attempt in range(_MAX_ATTEMPTS):
             # Attempt first, sleep only between attempts: a trace that is already ingested
             # when we look must cost nothing, which is the common case once linking is
             # batched to the end of the run.
             try:
-                found = _fetch_traces_for_session(langfuse, cid, window_start, window_end, pad)
+                found = fetch(langfuse, cid, window_start, window_end, pad)
             except Exception as exc:
                 _log.debug("Langfuse trace fetch failed for %s: %s", cid, exc)
             if found or time.monotonic() + delay > stop_at:
@@ -298,11 +338,19 @@ def _experiment_root_span(
     conversation_id: str | None,
     item_input: Any,
     output: Any,
+    joined_ids: tuple[str, str] | None = None,
 ) -> Span:
-    """gd-eval's own root span for one (dataset item, run) -- the whole experiment item."""
+    """gd-eval's own root span for one (dataset item, run) -- the whole experiment item.
+
+    ``joined_ids`` is the ``(trace_id, span_id)`` gen-ai's turns were put under; the span then
+    takes those ids and the environment the langfuse SDK forced on those turns.
+    """
     start, end = _span_window(trace, window)
     session_id = conversation_id or getattr(trace, "session_id", None)
-    tags = tuple(tag for tag in ("gd-eval", run_metadata.get("testing_framework")) if tag)
+    custom_tags = run_metadata.get("tags") or ()
+    if isinstance(custom_tags, str):
+        custom_tags = (custom_tags,)
+    tags = trace_tags("gd-eval", run_metadata.get("testing_framework"), *custom_tags)
     # One predicate for both the name and the input, so a falsy question cannot name the span
     # after the item while the input still says "question".
     has_input = item_input is not None
@@ -328,7 +376,9 @@ def _experiment_root_span(
             "conversation_id": session_id,
         },
         trace_metadata={"run_name": run_name},
-        environment=os.environ.get("LANGFUSE_TRACING_ENVIRONMENT"),
+        environment=SDK_EXPERIMENT_ENVIRONMENT if joined_ids else os.environ.get("LANGFUSE_TRACING_ENVIRONMENT"),
+        trace_id=joined_ids[0] if joined_ids else None,
+        span_id=joined_ids[1] if joined_ids else None,
     )
 
 
@@ -377,6 +427,22 @@ def observe(
         yield None
         return
 
+    joined = joined_run(conversation_id)
+    if joined is not None and conversation_id:
+        yield _export_joined_root(
+            langfuse,
+            joined,
+            conversation_id,
+            dataset_item_id,
+            run_name,
+            run_metadata or {},
+            trace=trace,
+            window=window,
+            item_input=item_input,
+            output=output,
+        )
+        return
+
     try:
         dataset_id = langfuse.dataset_id_for_item(dataset_item_id)
     except Exception as exc:
@@ -431,6 +497,85 @@ def observe(
     yield ScoreTarget(trace_id, span.trace_id, span.span_id)
 
 
+def _export_joined_root(
+    langfuse: HttpxLangfuseClient,
+    joined: JoinedRun,
+    conversation_id: str,
+    dataset_item_id: str,
+    run_name: str,
+    run_metadata: dict[str, Any],
+    *,
+    trace: Any,
+    window: tuple[datetime, datetime] | None,
+    item_input: Any,
+    output: Any,
+) -> ScoreTarget:
+    """Export the root of a joined conversation's trace and return its one score destination.
+
+    The root takes the trace and span ids gen-ai's turns were sent, and the experiment run
+    they were told they belong to, so Langfuse assembles one experiment item from both.
+    ``trace`` is that trace as read before scoring; its window spans every gen-ai row.
+    """
+    trace_id, span_id = trace_ids_for(conversation_id)
+    if joined.run_name != run_name:
+        _log.warning(
+            "Conversation %s joined run %s, not %s; its root follows the join.",
+            conversation_id,
+            joined.run_name,
+            run_name,
+        )
+    span = _experiment_root_span(
+        trace_id,
+        dataset_item_id,
+        joined.dataset_id,
+        joined.run_name,
+        run_metadata,
+        trace=trace,
+        window=window,
+        conversation_id=conversation_id,
+        item_input=item_input,
+        output=output,
+        joined_ids=(trace_id, span_id),
+    )
+    try:
+        langfuse.export_spans([span])
+    except Exception as exc:
+        _log.warning("Failed to export the experiment span for run %s: %s", joined.run_name, exc)
+        warn_from_worker(
+            f"[langfuse] WARNING: failed to export experiment span run={joined.run_name} item={dataset_item_id}: {exc}"
+        )
+        return ScoreTarget(trace_id)
+    finally:
+        release_joined(conversation_id)
+    return ScoreTarget(experiment_trace_id=trace_id, experiment_span_id=span_id)
+
+
+# Scores held by an open ``collect_scores`` block on this thread, sent together when it ends.
+_PENDING_SCORES: contextvars.ContextVar[list[dict[str, Any]] | None] = contextvars.ContextVar(
+    "gd_eval_pending_scores", default=None
+)
+
+
+@contextmanager
+def collect_scores(langfuse: HttpxLangfuseClient) -> Iterator[None]:
+    """Hold every ``score_safe`` write made in the block and send them as one batch at its end.
+
+    The batch is sent even when the drain was cancelled meanwhile, without retrying, so the
+    scores made before the interrupt are kept.
+    """
+    pending: list[dict[str, Any]] = []
+    token = _PENDING_SCORES.set(pending)
+    try:
+        yield
+    finally:
+        _PENDING_SCORES.reset(token)
+        if pending:
+            try:
+                langfuse.create_scores(pending, wait=_wait_between_attempts)
+            except Exception as exc:
+                _log.warning("Failed to log %d scores: %s", len(pending), exc)
+
+
 def score_safe(langfuse: Any, trace_id: Any, **kwargs: Any) -> None:
     """Create one Langfuse score per destination the target names, ignoring errors.
 
@@ -444,7 +589,11 @@ def score_safe(langfuse: Any, trace_id: Any, **kwargs: Any) -> None:
     if _drain_is_cancelled():
         return
     targets = trace_id.destinations() if isinstance(trace_id, ScoreTarget) else [(str(trace_id), None)]
+    pending = _PENDING_SCORES.get()
     for target_id, observation_id in targets:
+        if pending is not None:
+            pending.append({"trace_id": target_id, "observation_id": observation_id, **kwargs})
+            continue
         extra = {"observation_id": observation_id} if observation_id else {}
         try:
             langfuse.create_score(trace_id=target_id, **kwargs, **extra)
@@ -493,6 +642,42 @@ def log_quality_and_value_scores(
     )
 
 
+def run_context_for(identity: RunIdentity) -> tuple[str, dict[str, Any]]:
+    """``build_run_context`` for an item's ``RunIdentity``."""
+    return build_run_context(
+        identity.host,
+        identity.token,
+        identity.workspace_id,
+        identity.dataset_name,
+        identity.run_timestamp,
+        identity.model_version_override,
+        identity.run_metadata_extra,
+        identity.reasoning_effort,
+    )
+
+
+def resolve_item_scope(
+    langfuse: Any, identity: RunIdentity, dataset_item_id: str, *, suffix_runs: bool
+) -> ItemTraceScope | None:
+    """The item's experiment scope, or None when ``observe`` could not export its root.
+
+    That is: a client other than ``HttpxLangfuseClient``, linking switched off, or a dataset
+    item Langfuse does not know. Sending the join keys then would leave gen-ai's turns under
+    a root that never arrives.
+    """
+    if not isinstance(langfuse, HttpxLangfuseClient) or env_flag(SKIP_ENV_VAR):
+        return None
+    try:
+        dataset_id = langfuse.dataset_id_for_item(dataset_item_id)
+    except Exception as exc:
+        _log.warning("Failed to resolve dataset item %s; its chat turns will not join: %s", dataset_item_id, exc)
+        return None
+    if dataset_id is None:
+        return None
+    base_name, run_metadata = run_context_for(identity)
+    return ItemTraceScope(base_name, suffix_runs, dataset_id, dataset_item_id, run_metadata)
+
+
 def build_run_context(
     host: str,
     token: str,
@@ -514,7 +699,8 @@ def build_run_context(
         model_version_override: Explicit model-version tag; when set it wins over the
             workspace lookup.
         run_metadata_extra: Optional extra key/values added to the dataset-run metadata
-            (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps
+            (e.g. a testing-framework tag or a CI run id for scoping). A `tags` list is
+            also added to the trace tags, after the automatic ones. Default None keeps
             behavior unchanged. The SDK-derived model_version is applied last and cannot
             be overwritten by this dict.
         reasoning_effort: Effort the run requested, stamped into both the run name and