gooddata-eval 1.79.1.dev1__py3-none-any.whl → 1.79.2.dev1__py3-none-any.whl

gooddata_eval/core/agentic/_clarification.py

@@ -0,0 +1,511 @@
+# (C) 2026 GoodData Corporation. All rights reserved.
+"""Whether a conversation turn's first reply asked the clarifying question its fixture expects."""
+
+from __future__ import annotations
+
+import functools
+import re
+from collections.abc import Callable
+from typing import Any, Literal
+
+from gooddata_api_client.exceptions import ApiException, NotFoundException
+from gooddata_sdk import GoodDataSdk
+from gooddata_sdk.utils import load_all_entities
+from pydantic import BaseModel, Field, model_validator
+from urllib3.exceptions import HTTPError
+
+from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, retry_transient
+from gooddata_eval.core.models import ChatResult
+
+ClarificationForm = Literal["questionnaire", "text"]
+
+_REF_RE = re.compile(r"\{([A-Za-z_]+)/([^{}]+)\}")
+# The entity kinds a `{type/id}` option reference can name. An attribute and its label are
+# spelled either way by the agent, so each accepts the other. A date dataset is a dataset.
+_REF_ENTITIES: dict[str, tuple[str, ...]] = {
+    "metric": ("metrics",),
+    "fact": ("facts",),
+    "dataset": ("datasets",),
+    "date": ("datasets",),
+    "attribute": ("attributes", "labels"),
+    "date_attribute": ("attributes", "labels"),
+    "label": ("labels", "attributes"),
+    "parameter": ("parameters",),
+    "dashboard": ("analyticalDashboards",),
+    "analyticalDashboard": ("analyticalDashboards",),
+    "visualization": ("visualizationObjects",),
+    "visualizationObject": ("visualizationObjects",),
+    "insight": ("visualizationObjects",),
+}
+# Reference types gen-ai writes that no entity listing holds, so grounding cannot check them.
+_UNLISTED_REF_TYPES = frozenset({"computed_attribute", "filter_context", "document", "report"})
+_ENTITY_KINDS = tuple(sorted({kind for kinds in _REF_ENTITIES.values() for kind in kinds}))
+_PAGE_SIZE = 1000
+# The EntitiesApi listing of each entity kind.
+_LIST_METHODS: dict[str, str] = {
+    "analyticalDashboards": "get_all_entities_analytical_dashboards",
+    "attributes": "get_all_entities_attributes",
+    "datasets": "get_all_entities_datasets",
+    "facts": "get_all_entities_facts",
+    "labels": "get_all_entities_labels",
+    "metrics": "get_all_entities_metrics",
+    "parameters": "get_all_entities_parameters",
+    "visualizationObjects": "get_all_entities_visualization_objects",
+}
+# A shorter skipped option, such as "No" or "Q1", also occurs in text that assumes nothing, so it
+# never counts as named.
+_MIN_NAMED_OPTION_CHARS = 3
+
+
+OptionTitleKind = Literal["metric", "attribute", "fact", "dataset", "dashboard", "visualization"]
+_TITLE_ENTITIES: dict[str, str] = {
+    "metric": "metrics",
+    "attribute": "attributes",
+    "fact": "facts",
+    "dataset": "datasets",
+    "dashboard": "analyticalDashboards",
+    "visualization": "visualizationObjects",
+}
+# What may follow a title in an option that adds context, as two charts with one title need.
+_TITLE_CONTEXT_SEPARATORS = (" (", " [", " - ", " – ", " — ", ": ", ", ")
+_GENERIC_ANSWERS = frozenset(
+    {"yes", "no", "ok", "okay", "cancel", "confirm", "continue", "proceed", "skip", "not now", "go ahead"}
+)
+# How an option may spell a chart type whose `local:<type>` id is not a word of its own.
+_CHART_TYPE_WORDS: dict[str, tuple[str, ...]] = {
+    "combo2": ("combo",),
+    "pushpin": ("geo", "map", "pushpin"),
+    "heatmap": ("heatmap", "heat map"),
+    "treemap": ("treemap", "tree map"),
+    "dependencywheel": ("dependency wheel",),
+}
+
+
+class QuestionOptions(BaseModel):
+    """One question's options: the ``type/id`` references they name, and the options that name none."""
+
+    refs: list[str] = Field(default_factory=list)
+    plain: list[str] = Field(default_factory=list)
+
+
+class FirstReply(BaseModel):
+    """What the turn's first assistant reply asked, read before any simulated-user answer."""
+
+    asked: bool
+    form: ClarificationForm | None = None
+    questions: list[QuestionOptions] = Field(default_factory=list)
+
+    @property
+    def question_count(self) -> int:
+        return len(self.questions)
+
+    @property
+    def option_refs(self) -> list[str]:
+        return [ref for q in self.questions for ref in q.refs]
+
+    @property
+    def plain_options(self) -> list[str]:
+        return [option for q in self.questions for option in q.plain]
+
+    @property
+    def has_options(self) -> bool:
+        return any(q.refs or q.plain for q in self.questions)
+
+
+def read_first_reply(chat_result: ChatResult, *, asked_in_text: bool) -> FirstReply:
+    """The reply's clarifying questions; a ``clarifyingQuestions`` part makes it a questionnaire.
+
+    ``asked_in_text`` is the caller's reading of the prose, for a question without a part.
+    """
+    parts = [p for p in chat_result.unhandled_parts or [] if p.get("type") == "clarifyingQuestions"]
+    if not parts:
+        return FirstReply(asked=asked_in_text, form="text" if asked_in_text else None)
+    questions = []
+    for question in (q for p in parts for q in p.get("questions") or []):
+        options = _options(question)
+        questions.append(
+            QuestionOptions(
+                refs=[f"{kind}/{obj_id}" for text in options for kind, obj_id in _REF_RE.findall(text)],
+                plain=[text for text in options if not _REF_RE.search(text)],
+            )
+        )
+    return FirstReply(asked=True, form="questionnaire", questions=questions)
+
+
+class WorkspaceObjects(BaseModel):
+    """The workspace's objects as the option checks read them: ``<entity kind>/<id>`` and their titles."""
+
+    refs: set[str] = Field(default_factory=set)
+    # `<entity kind>/<id>` -> the title, casefolded with whitespace collapsed.
+    titles: dict[str, str] = Field(default_factory=dict)
+    # `visualizationObjects/<id>` -> the chart type, `local:donut` read as `donut`.
+    chart_types: dict[str, str] = Field(default_factory=dict)
+
+    def titles_of(self, entity_kind: str) -> set[str]:
+        return {title for ref, title in self.titles.items() if ref.startswith(f"{entity_kind}/")}
+
+
+def _title_key(text: str) -> str:
+    return " ".join(text.casefold().split())
+
+
+def _is_transient(exc: Exception) -> bool:
+    if isinstance(exc, ApiException):
+        return exc.status in RETRYABLE_STATUS_CODES
+    return isinstance(exc, HTTPError)
+
+
+def _page_with_retry(list_page: Callable[..., Any], **page: int) -> Any:
+    return retry_transient(lambda: list_page(**page), is_retryable=_is_transient)
+
+
+def fetch_workspace_objects(sdk: GoodDataSdk, workspace_id: str) -> WorkspaceObjects:
+    """Every ``<entity kind>/<id>`` and title the workspace holds, inherited objects included.
+
+    A page that fails with a transient error is retried as a chat message is. A kind the server does
+    not know (404) has no objects. Any other failure of the entity API is raised.
+    """
+    found = WorkspaceObjects()
+    api = sdk._client.entities_api
+    for kind in _ENTITY_KINDS:
+        list_page = functools.partial(getattr(api, _LIST_METHODS[kind]), workspace_id, _check_return_type=False)
+        try:
+            data = load_all_entities(functools.partial(_page_with_retry, list_page), page_size=_PAGE_SIZE).data
+        except NotFoundException:
+            continue
+        for obj in data:
+            if not isinstance(obj, dict) or not obj.get("id"):
+                continue
+            ref = f"{kind}/{obj['id']}"
+            found.refs.add(ref)
+            raw_attributes = obj.get("attributes")
+            attributes: dict[str, Any] = raw_attributes if isinstance(raw_attributes, dict) else {}
+            title = attributes.get("title")
+            if isinstance(title, str) and title:
+                found.titles[ref] = _title_key(title)
+            raw_content = attributes.get("content")
+            content: dict[str, Any] = raw_content if isinstance(raw_content, dict) else {}
+            chart_url = str(content.get("visualizationUrl") or "")
+            if chart_url.startswith("local:"):
+                found.chart_types[ref] = chart_url.removeprefix("local:")
+    return found
+
+
+def ungrounded_refs(refs: list[str], known: set[str]) -> list[str]:
+    """The references that name no object of the workspace, or a kind it has none of.
+
+    A type of ``_UNLISTED_REF_TYPES`` is left unchecked.
+    """
+    missing: list[str] = []
+    for ref in refs:
+        ref_type, _, obj_id = ref.partition("/")
+        if ref_type in _UNLISTED_REF_TYPES:
+            continue
+        kinds = _REF_ENTITIES.get(ref_type, ())
+        if not any(f"{kind}/{obj_id}" in known for kind in kinds):
+            missing.append(ref)
+    return missing
+
+
+def _is_titled(option: str, titles: set[str]) -> bool:
+    key = _title_key(option)
+    return key in titles or any(key.startswith(title + sep) for title in titles for sep in _TITLE_CONTEXT_SEPARATORS)
+
+
+def untitled_options(options: list[str], objects: WorkspaceObjects, kind: OptionTitleKind) -> list[str]:
+    """The options that are not the title of a ``kind`` object, alone or followed by added context."""
+    titles = objects.titles_of(_TITLE_ENTITIES[kind])
+    return [option for option in options if not _is_titled(option, titles)]
+
+
+def _off_in_scope(
+    questions: list[QuestionOptions], off_of: Callable[[QuestionOptions], tuple[int, list[str]]]
+) -> list[str]:
+    """The options an option check fails, read only from the questions the check is about.
+
+    ``off_of`` gives a question's count of options the check reads and the ones it fails. A questionnaire
+    can ask about more than one thing, such as a metric next to a channel: a question with at least one
+    option the check accepts is about it, and a question with none is about something else and left out.
+    When no question is about it, every option the check read fails.
+    """
+    checked = [off_of(q) for q in questions]
+    about = [off for read, off in checked if len(off) < read]
+    return [option for off in (about or [off for _, off in checked]) for option in off]
+
+
+def untitled_question_options(reply: FirstReply, objects: WorkspaceObjects, kind: OptionTitleKind) -> list[str]:
+    """The plain-text options that are not the title of a ``kind`` object, in the questions about that kind."""
+    return _off_in_scope(reply.questions, lambda q: (len(q.plain), untitled_options(q.plain, objects, kind)))
+
+
+def _entity_refs(ref: str) -> set[str]:
+    ref_type, _, obj_id = ref.partition("/")
+    return {f"{kind}/{obj_id}" for kind in _REF_ENTITIES.get(ref_type, ())}
+
+
+def is_object_ref(ref: str) -> bool:
+    """Whether ``ref`` is ``type/id`` with a type an option reference can name."""
+    ref_type, sep, obj_id = ref.partition("/")
+    return bool(sep and obj_id) and ref_type in _REF_ENTITIES
+
+
+def _names_chart_type(option: str, chart_type: str) -> bool:
+    key = _title_key(option)
+    return any(re.search(rf"\b{re.escape(word)}\b", key) for word in _CHART_TYPE_WORDS.get(chart_type, (chart_type,)))
+
+
+def off_candidate_options(reply: FirstReply, objects: WorkspaceObjects, candidates: list[str]) -> list[str]:
+    """The options that name none of ``candidates``, in the questions that offer at least one of them.
+
+    An option names a candidate by reference, by its title plus context, or by its chart type when
+    no other candidate has that type: "Donut chart" tells two same-titled charts apart only then.
+    A chart type does not count in an option that starts with the title of an object outside the candidates.
+    """
+    wanted = set().union(*(_entity_refs(c) for c in candidates))
+    titles = {objects.titles[ref] for ref in wanted if ref in objects.titles}
+    kinds = {ref.partition("/")[0] for ref in wanted}
+    other_titles = {
+        title for ref, title in objects.titles.items() if ref not in wanted and ref.partition("/")[0] in kinds
+    } - titles
+    types = [objects.chart_types[ref] for ref in wanted if ref in objects.chart_types]
+    unique_types = {chart_type for chart_type in types if types.count(chart_type) == 1}
+
+    def names_a_candidate_type(option: str) -> bool:
+        return any(_names_chart_type(option, t) for t in unique_types) and not _is_titled(option, other_titles)
+
+    def off_of(question: QuestionOptions) -> tuple[int, list[str]]:
+        off = [ref for ref in question.refs if not (_entity_refs(ref) & wanted)]
+        off += [
+            option for option in question.plain if not _is_titled(option, titles) and not names_a_candidate_type(option)
+        ]
+        return len(question.refs) + len(question.plain), off
+
+    return _off_in_scope(reply.questions, off_of)
+
+
+def clarification_mismatches(
+    reply: FirstReply,
+    *,
+    expects_clarification: bool,
+    expected_form: ClarificationForm | None,
+    max_questions: int | None,
+    objects: WorkspaceObjects | None,
+    grounded: bool,
+    titled_as: OptionTitleKind | None,
+    candidates: list[str] | None = None,
+    reads_questionnaire: bool = False,
+) -> list[str]:
+    """How the first reply differs from the expected clarification; empty when it matches.
+
+    ``grounded`` checks the ``{type/id}`` options of every question; ``titled_as`` checks the plain-text
+    ones against that kind's titles; ``candidates`` requires every option to name one of those objects.
+    The last two read only the questions they are about (see ``_off_in_scope``). All three are skipped
+    when ``objects`` is None: the caller reports the turn ungraded then.
+    ``reads_questionnaire`` fails a prose question, which has no options for these checks to read.
+    """
+    if not expects_clarification:
+        return [f"asked a clarifying question ({reply.form}) the request does not need"] if reply.asked else []
+    if not reply.asked:
+        return ["expected a clarifying question, but the first reply asked none"]
+    out: list[str] = []
+    if expected_form is not None and reply.form != expected_form:
+        out.append(f"clarification form: expected {expected_form}, got {reply.form}")
+    elif reads_questionnaire and reply.form != "questionnaire":
+        out.append(f"the turn checks a questionnaire, but the first reply asked as {reply.form}")
+    if max_questions is not None and reply.question_count > max_questions:
+        out.append(f"asked {reply.question_count} questions, at most {max_questions} expected")
+    if objects is not None and grounded:
+        missing = ungrounded_refs(reply.option_refs, objects.refs)
+        if missing:
+            out.append(f"options name objects the workspace does not have: {missing}")
+    if objects is not None and titled_as is not None:
+        untitled = untitled_question_options(reply, objects, titled_as)
+        if untitled:
+            out.append(f"options are not the title of a {titled_as} of the workspace: {untitled}")
+    if objects is not None and candidates:
+        off = off_candidate_options(reply, objects, candidates)
+        if off:
+            out.append(f"options name none of {candidates}: {off}")
+    return out
+
+
+class ClarifyingPick(BaseModel):
+    """An option the fixture picks in a questionnaire.
+
+    ``option`` is a ``type/id`` reference the option names, or the option's whole text. A list names
+    the spellings of one option, since the agent writes it as a reference or as plain text.
+    ``question`` limits the pick to that position; without it, the first question offering the
+    option takes it.
+    """
+
+    option: str | list[str]
+    question: int | None = Field(default=None, ge=0)
+
+    @property
+    def spellings(self) -> list[str]:
+        return [self.option] if isinstance(self.option, str) else list(self.option)
+
+    @property
+    def label(self) -> str:
+        return " | ".join(self.spellings)
+
+    @model_validator(mode="after")
+    def _names_an_option(self) -> ClarifyingPick:
+        if not self.spellings or not all(s.strip() for s in self.spellings):
+            raise ValueError("option needs at least one non-empty spelling")
+        return self
+
+
+class ClarifyingAnswers(BaseModel):
+    """The ``clarifyingAnswers`` for one questionnaire, and what they left out."""
+
+    answers: list[dict] = Field(default_factory=list)
+    skipped: list[dict] = Field(default_factory=list)
+    matched_picks: list[str] = Field(default_factory=list)
+
+    def render(self) -> str:
+        """The text gen-ai builds for the model from these answers: every question, a skipped one as Skipped."""
+        sections = [(a["questionIndex"], a["questionText"], a["answer"]["optionText"]) for a in self.answers]
+        sections += [(s["index"], s["text"], "Skipped") for s in self.skipped]
+        return "\n\n".join(f"**{text}**\n\n{answer}" for _, text, answer in sorted(sections, key=lambda s: s[0]))
+
+
+def _all_questions(chat_result: ChatResult) -> list[dict]:
+    parts = [p for p in chat_result.unhandled_parts or [] if p.get("type") == "clarifyingQuestions"]
+    return [q for p in parts for q in p.get("questions") or []]
+
+
+def _questions(chat_result: ChatResult) -> list[dict]:
+    """The questions of the item the answers name, numbered as gen-ai numbers them: across that item's parts."""
+    item_id = chat_result.clarifying_questions_item_id
+    parts = [
+        p
+        for p in chat_result.unhandled_parts or []
+        if p.get("type") == "clarifyingQuestions" and p.get("itemId") in (None, item_id)
+    ]
+    return [q for p in parts for q in p.get("questions") or []]
+
+
+def _options(question: dict) -> list[str]:
+    return [str(o.get("text") or "") for o in (question.get("control") or {}).get("options") or []]
+
+
+def _plain(text: str) -> str:
+    return " ".join(_REF_RE.sub(" ", text).lower().split())
+
+
+def _same_object(ref: str, other: str) -> bool:
+    return ref == other or bool(_entity_refs(ref) & _entity_refs(other))
+
+
+def option_matches(option_text: str, wanted: str) -> bool:
+    """Whether an option is the one a pick names, by its ``{type/id}`` reference or its whole text.
+
+    A reference matches the spellings grounding accepts for the same object, such as ``attribute/x`` and ``label/x``.
+    """
+    ref = wanted.strip().strip("{}")
+    if "/" in ref and any(_same_object(ref, f"{kind}/{obj_id}") for kind, obj_id in _REF_RE.findall(option_text)):
+        return True
+    return option_text.strip().lower() == wanted.strip().lower()
+
+
+def build_clarifying_answers(chat_result: ChatResult, picks: list[ClarifyingPick]) -> ClarifyingAnswers:
+    """Answer the reply's questionnaire with the picks; a question no pick matches is skipped."""
+    questions = _questions(chat_result)
+    out = ClarifyingAnswers()
+    answered: set[int] = set()
+    for pick in picks:
+        for qi, question in enumerate(questions):
+            if qi in answered or (pick.question is not None and pick.question != qi):
+                continue
+            oi = next(
+                (
+                    i
+                    for i, text in enumerate(_options(question))
+                    if any(option_matches(text, spelling) for spelling in pick.spellings)
+                ),
+                None,
+            )
+            if oi is None:
+                continue
+            answered.add(qi)
+            out.matched_picks.append(pick.label)
+            out.answers.append(
+                {
+                    "questionIndex": qi,
+                    "questionText": question.get("text") or "",
+                    "answer": {"type": "singleChoice", "optionIndex": oi, "optionText": _options(question)[oi]},
+                }
+            )
+            break
+    out.answers.sort(key=lambda a: a["questionIndex"])
+    out.skipped = [
+        {"index": qi, "text": q.get("text") or "", "options": _options(q)}
+        for qi, q in enumerate(questions)
+        if qi not in answered
+    ]
+    return out
+
+
+def _names_option(text: str, option: str) -> bool:
+    """Whether ``text`` names the option: one of its references whole, or its words as whole words."""
+    refs_in_text = [f"{kind}/{obj_id}" for kind, obj_id in _REF_RE.findall(text)]
+    for kind, obj_id in _REF_RE.findall(option):
+        if any(_same_object(f"{kind}/{obj_id}", ref) for ref in refs_in_text):
+            return True
+    label = _plain(option)
+    return (
+        len(label) >= _MIN_NAMED_OPTION_CHARS
+        and re.search(rf"(?<!\w){re.escape(label)}(?!\w)", _plain(text)) is not None
+    )
+
+
+def _same_question(question: dict, skipped: dict) -> bool:
+    """Whether ``question`` asks ``skipped`` again: the same text, or the same options that name something.
+
+    Options such as Yes and No fit any question, a confirmation too, so they count only with the same text.
+    """
+    if _plain(question.get("text") or "") == _plain(skipped["text"]):
+        return True
+    options = sorted(_options(question))
+    specific = any(_plain(option).strip(".!") not in _GENERIC_ANSWERS for option in options)
+    return specific and options == sorted(skipped["options"])
+
+
+def after_answers_mismatches(
+    *,
+    unmatched_picks: list[str],
+    skipped: list[dict],
+    replies_after: list[tuple[str, ChatResult, bool]],
+    states_skipped_assumption: bool,
+    skipped_questions_not_reasked: bool,
+    expects_confirmation_after_answers: bool,
+) -> list[str]:
+    """How the turn after its structured answers differs from the fixture; empty when it matches.
+
+    ``replies_after`` holds each later reply with its kind (``output`` or a ``classify_reply`` kind) and
+    whether it answers a ``clarifyingAnswers`` message directly.
+
+    ``states_skipped_assumption`` stands in for "the agent says what it assumed": it passes when any reply
+    after the answers names an option of each skipped question. It cannot tell an assumption from a
+    mention, and it does not need the assumption in the final reply, since a reply that asks for
+    confirmation usually states it.
+    """
+    out = [f"no questionnaire offered the picked option {pick!r}" for pick in unmatched_picks]
+    if states_skipped_assumption:
+        texts = [reply.text_response or "" for _, reply, _ in replies_after]
+        out.extend(
+            f"no reply after the answers names an option of the skipped question {question['text']!r}"
+            for question in skipped
+            if not any(_names_option(text, option) for text in texts for option in question["options"])
+        )
+    if skipped_questions_not_reasked:
+        for question in skipped:
+            asked_again = any(
+                _same_question(later, question) for _, reply, _ in replies_after for later in _all_questions(reply)
+            )
+            if asked_again:
+                out.append(f"asked the skipped question {question['text']!r} again")
+    if expects_confirmation_after_answers and any(kind == "output" and follows for kind, _, follows in replies_after):
+        out.append("acted on the answers in the same reply, without waiting for confirmation")
+    return out

gooddata_eval/core/agentic/_conversation_context.py

@@ -68,6 +68,8 @@ _RETRY_CHOICE_RE = re.compile(
 # word, so the apostrophes in "don't ... it's" do not mask the text between them.
 _FORMULA_SPAN_RE = re.compile(r"`+[^`]*`+|\{[^{}]*\}|\"[^\"\n]*\"|(?<!\w)'[^'\n]*'(?!\w)")
 
+# Where one sentence ends and the next starts, for asks_in_text.
+_SENTENCE_END_RE = re.compile(r"(?<=[.?!])\s+")
 # Checked after _CONFIRMATION_RE, so "please confirm" stays a confirmation.
 _REQUEST_RE = re.compile(
     r"\bplease\s+(?:select|choose|pick|specify|provide|tell\s+me|let\s+me\s+know|clarify|share)\b"
@@ -106,6 +108,20 @@ def classify_reply(chat_result: ChatResult, rendered_text: str) -> ReplyKind:
     return "no_action"
 
 
+def asks_in_text(chat_result: ChatResult, rendered_text: str) -> bool:
+    """Whether a reply without the expected output asks the user something, read for the clarification checks.
+
+    ``classify_reply`` reads "Did you mean A or B? Please confirm." as a confirmation, which is right for
+    what the simulated user says back. Here it is a question: a sentence that ends in a question mark and
+    is not itself a confirmation phrase asks, whatever confirmation follows it.
+    """
+    kind = classify_reply(chat_result, rendered_text)
+    if kind != "confirmation" or chat_result.alert_proposals:
+        return kind == "question"
+    sentences = _SENTENCE_END_RE.split(rendered_text.strip())
+    return any(s.rstrip().endswith("?") and not _CONFIRMATION_RE.search(s) for s in sentences)
+
+
 @dataclass
 class TranscriptEntry:
     role: Literal["user", "assistant"]

gooddata_eval/core/agentic/conversation.py

@@ -27,13 +27,28 @@ from typing import ClassVar, Literal
 from gooddata_sdk import GoodDataSdk
 from pydantic import BaseModel, Field, ValidationError, model_validator
 
+from gooddata_eval.core.agentic._clarification import (
+    ClarificationForm,
+    ClarifyingPick,
+    FirstReply,
+    OptionTitleKind,
+    WorkspaceObjects,
+    after_answers_mismatches,
+    build_clarifying_answers,
+    clarification_mismatches,
+    fetch_workspace_objects,
+    is_object_ref,
+    read_first_reply,
+)
 from gooddata_eval.core.agentic._conversation_context import (
     CONFIRMATION_REPLY,
     NUDGE_MESSAGE,
     ClarificationJudge,
     TranscriptEntry,
+    asks_in_text,
     classify_reply,
     clip,
+    has_structured_question,
     judge_prompt,
     render_transcript,
     reply_from_facts,
@@ -108,6 +123,27 @@ class TurnDefinition(BaseModel):
     and setting it to null clears it. A turn that leaves it out keeps the current one. Its
     clarification replies carry it too, since gen-ai treats a message without a context as
     cleared.
+
+    ``expects_clarification`` grades the turn's first reply: true means it must ask, false means
+    it must not. Left out, nothing is checked. With true, ``expected_clarification_form`` names how
+    it asks (``questionnaire`` = a ``clarifyingQuestions`` part, ``text`` = prose only),
+    ``max_clarifying_questions`` caps the questionnaire, and ``clarifying_options_grounded``
+    requires every ``{type/id}`` option to name an object of the workspace. ``clarifying_option_titles``
+    names an object kind (``visualization``, ``metric``, ...): every plain-text option must be the title
+    of such an object, alone or followed by context in brackets or after a dash, comma or colon.
+    ``clarifying_option_objects`` lists the ``type/id`` objects the question is about, such as the
+    charts sharing the title the user named: every option must name one of them, by reference, by
+    its title with that context, or by its chart type when no other candidate has that type.
+
+    ``clarifying_answers`` answers the questionnaire of the turn's first reply the way the chat panel
+    does, with a ``clarifyingAnswers`` message: a question no pick matches is skipped, and an empty
+    list skips them all. Every other question, a later questionnaire included, gets ``set_answers``
+    text, so a follow-up confirmation can be answered. With it,
+    ``states_skipped_assumption`` requires a reply after the answers to name an option of each skipped
+    question (a mention, which stands in for the stated assumption),
+    ``skipped_questions_not_reasked`` forbids a later questionnaire asking one again, and
+    ``expects_confirmation_after_answers`` requires the agent to stop after each ``clarifyingAnswers``
+    message rather than act on it in the same reply: a skip is not an approval.
     """
 
     turn_id: str
@@ -121,6 +157,66 @@ class TurnDefinition(BaseModel):
     depends_on: list[str] = Field(default_factory=list)
     set_answers: list[str] = Field(default_factory=list)
     user_context: dict | None = None
+    expects_clarification: bool | None = None
+    expected_clarification_form: ClarificationForm | None = None
+    max_clarifying_questions: int | None = Field(default=None, ge=1)
+    clarifying_options_grounded: bool = False
+    clarifying_option_titles: OptionTitleKind | None = None
+    clarifying_option_objects: list[str] | None = None
+    clarifying_answers: list[ClarifyingPick] | None = None
+    states_skipped_assumption: bool = False
+    skipped_questions_not_reasked: bool = False
+    expects_confirmation_after_answers: bool = False
+
+    @property
+    def reads_questionnaire(self) -> bool:
+        """Whether a clarification key of the turn reads a ``clarifyingQuestions`` part, so prose cannot satisfy it."""
+        return (
+            self.max_clarifying_questions is not None
+            or self.clarifying_options_grounded
+            or self.clarifying_option_titles is not None
+            or self.clarifying_option_objects is not None
+            or self.clarifying_answers is not None
+        )
+
+    @model_validator(mode="after")
+    def _clarification_keys_need_an_expected_question(self) -> TurnDefinition:
+        # Each of these only reads a question, so without one it would pass vacuously.
+        stated = [
+            name
+            for name, value in (
+                ("expected_clarification_form", self.expected_clarification_form),
+                ("max_clarifying_questions", self.max_clarifying_questions),
+                ("clarifying_options_grounded", self.clarifying_options_grounded or None),
+                ("clarifying_option_titles", self.clarifying_option_titles),
+                ("clarifying_option_objects", self.clarifying_option_objects),
+                ("clarifying_answers", self.clarifying_answers),
+            )
+            if value is not None
+        ]
+        if stated and self.expects_clarification is not True:
+            raise ValueError(f"{stated} need expects_clarification: true")
+        if self.reads_questionnaire and self.expected_clarification_form == "text":
+            raise ValueError(
+                "max_clarifying_questions, clarifying_options_grounded, clarifying_option_titles, "
+                "clarifying_option_objects and clarifying_answers read a questionnaire, not text"
+            )
+        after_answers = [
+            name
+            for name, value in (
+                ("states_skipped_assumption", self.states_skipped_assumption),
+                ("skipped_questions_not_reasked", self.skipped_questions_not_reasked),
+                ("expects_confirmation_after_answers", self.expects_confirmation_after_answers),
+            )
+            if value
+        ]
+        if self.clarifying_option_objects is not None:
+            bad = [ref for ref in self.clarifying_option_objects if not is_object_ref(ref)]
+            if not self.clarifying_option_objects or bad:
+                raise ValueError(f"clarifying_option_objects needs type/id references, got {bad or 'none'}")
+        if after_answers and self.clarifying_answers is None:
+            raise ValueError(f"{after_answers} read the turn after its clarifying_answers; set clarifying_answers")
+        return self
 
 
 class ConversationFixture(BaseModel):
@@ -196,6 +292,11 @@ class TurnResult(BaseModel):
     mismatches: list[str] = Field(default_factory=list)
     clarifications: list[ClarificationRecord] = Field(default_factory=list)
     depends_on: list[str] = Field(default_factory=list)
+    # None when the fixture states no expects_clarification, or the check could not run.
+    clarification_correct: bool | None = None
+    clarification_mismatches: list[str] = Field(default_factory=list)
+    # Why a stated clarification check could not run, so an ungraded turn does not read as a pass.
+    clarification_ungraded: str | None = None
 
     @property
     def skill_success(self) -> bool:
@@ -211,7 +312,13 @@ class TurnResult(BaseModel):
 
     @property
     def context_success(self) -> bool:
-        return self.skill_success and self.output_correct is not False and not self.lost_context and not self.stalled
+        return (
+            self.skill_success
+            and self.output_correct is not False
+            and self.clarification_correct is not False
+            and not self.lost_context
+            and not self.stalled
+        )
 
     def failure_reasons(self) -> list[str]:
         """Why ``context_success`` is False, in reading order; empty when it is True."""
@@ -224,6 +331,8 @@ class TurnResult(BaseModel):
             reasons.append(f"skill {self.expected_skill!r} not active")
         if self.output_correct is False:
             reasons.extend(self.mismatches or ["output does not match the expected output"])
+        if self.clarification_correct is False:
+            reasons.extend(self.clarification_mismatches)
         if self.lost_context:
             reasons.append("asked for information the conversation already established")
         if self.stalled:
@@ -258,6 +367,13 @@ class TurnResult(BaseModel):
         out = self.model_dump(include=self._DETAIL_FIELDS)
         out["context_success"] = self.context_success
         out["failure_reasons"] = self.failure_reasons()
+        # Only when the fixture states the check, so an item without one reports what it always did.
+        if self.clarification_correct is not None:
+            out["clarification_correct"] = self.clarification_correct
+            out["clarification_mismatches"] = list(self.clarification_mismatches)
+        elif self.clarification_ungraded is not None:
+            out["clarification_correct"] = None
+            out["clarification_ungraded"] = self.clarification_ungraded
         return out
 
 
@@ -669,6 +785,10 @@ class _WorkspaceObjects:
         self.updated_metrics: list[str] = []
         self.alerts: list[str] = []
 
+    def state(self) -> tuple[tuple[str, ...], ...]:
+        """What the conversation has written so far; a change means a workspace listing is out of date."""
+        return tuple(self.metrics), tuple(self.updated_metrics), tuple(self.alerts)
+
     def record(self, tool_call_events: list[ToolCallEvent]) -> None:
         for metric_id, created_new in _metric_creations(tool_call_events):
             if created_new:
@@ -733,8 +853,17 @@ def run_agentic_conversation(
 
     ``user_context`` is the context attached when the conversation starts; a turn's own
     ``user_context`` replaces it from that turn on.
+
+    A fixture with ``expects_clarification`` needs ``context`` mode: legacy mode neither sends
+    ``clarifying_answers`` nor lets the clarification checks fail the item, so it raises ``ValueError``.
     """
     resolved_mode = resolve_conversation_mode(mode)
+    graded = [t.turn_id for t in fixture.turns if t.expects_clarification is not None]
+    if resolved_mode == "legacy" and graded:
+        raise ValueError(
+            f"turns {graded} set expects_clarification, which only context mode grades; "
+            f"pass mode='context' or set {CONVERSATION_MODE_ENV_VAR}=context"
+        )
     if fresh_conversation_per_turn and initial_conversation_id is not None:
         raise ValueError("fresh_conversation_per_turn needs conversations this run creates itself")
     judge = clarification_judge if clarification_judge is not None else ClarificationJudge()
@@ -786,6 +915,9 @@ def run_agentic_conversation(
 
         run_started = time.monotonic()
         conversation_swap_s = 0.0  # fresh_conversation_per_turn's delete/create, which is not the run's work
+        workspace_read_s = 0.0  # the option checks' workspace listing, which is not the run's work either
+        # The option checks' listing, with the created.state() it was read at; read again once that changes.
+        workspace_listing: tuple[tuple[tuple[str, ...], ...], WorkspaceObjects] | None = None
         for turn_index, turn in enumerate(fixture.turns):
             if fresh_conversation_per_turn and turn_index > 0:
                 swap_started = time.monotonic()
@@ -842,6 +974,15 @@ def run_agentic_conversation(
             final_result: ChatResult | None = None
             records: list[ClarificationRecord] = []
             answers_left = list(turn.set_answers)
+            first_reply: FirstReply | None = None
+            # Structured answers: the one to send next, what they skipped, and every reply after them.
+            pending_answers: tuple[str, list[dict]] | None = None
+            answers_sent = False
+            # Why clarifying_answers were not sent, when the loop knows it better than what is left after.
+            answers_unsent: str | None = None
+            matched_picks: set[str] = set()
+            skipped_questions: list[dict] = []
+            replies_after_answers: list[tuple[str, ChatResult, bool]] = []
 
             # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop
             # that simply runs out of range() is labelled correctly with no trailing else.
@@ -849,8 +990,19 @@ def run_agentic_conversation(
             for _iter in range(max_clarification_turns + 1):
                 transcript.append(TranscriptEntry("user", current_message))
                 incomplete = False
+                follows_answers = pending_answers is not None
+                is_first_reply = False
                 try:
-                    chat_result = client.send_message(conversation_id, current_message, user_context=current_context)
+                    if pending_answers is not None:
+                        questions_item_id, answers = pending_answers
+                        pending_answers = None
+                        chat_result = client.send_clarifying_answers(
+                            conversation_id, questions_item_id, answers, user_context=current_context
+                        )
+                    else:
+                        chat_result = client.send_message(
+                            conversation_id, current_message, user_context=current_context
+                        )
                 except TurnIncompleteError as exc:
                     chat_result = exc.partial_result or ChatResult()
                     incomplete = True
@@ -910,10 +1062,30 @@ def run_agentic_conversation(
                         "assistant", " ".join(p for p in (response_text, summarize_visualizations(chat_result)) if p)
                     )
                 )
+                # A reply the server cut short is a stall the loop nudges past, so the next one is the first reply.
+                if first_reply is None and not incomplete:
+                    asked_in_text = not _check_output_present(resolved_turn, chat_result) and asks_in_text(
+                        chat_result, response_text
+                    )
+                    first_reply = read_first_reply(chat_result, asked_in_text=asked_in_text)
+                    is_first_reply = True
+                # The questionnaire clarifying_answers would answer, when this reply carries it.
+                answerable = (
+                    turn.clarifying_answers is not None and is_first_reply and has_structured_question(chat_result)
+                )
+                if answers_sent:
+                    reply_kind = (
+                        "output"
+                        if _check_output_present(resolved_turn, chat_result)
+                        else classify_reply(chat_result, response_text)
+                    )
+                    replies_after_answers.append((reply_kind, chat_result, follows_answers))
 
                 if _check_output_present(resolved_turn, chat_result):
                     if incomplete:
                         records.append(ClarificationRecord(kind="turn_incomplete", agent_message=clip(response_text)))
+                    if answerable:
+                        answers_unsent = "the first reply delivered the output with its questionnaire, so nothing waited on the answers"
                     turn_exit = LoopExit.SUCCESS
                     break
 
@@ -948,6 +1120,11 @@ def run_agentic_conversation(
                     and any(r.kind in ("no_action", "turn_incomplete") for r in records)
                 )
                 if clarification_turns >= max_clarification_turns or second_stall:
+                    if answerable:
+                        answers_unsent = (
+                            f"the clarification budget ({max_clarification_turns}) ended the turn "
+                            "before clarifying_answers were sent"
+                        )
                     records.append(record)
                     break
 
@@ -957,6 +1134,15 @@ def run_agentic_conversation(
                     reply = CONFIRMATION_REPLY
                 elif kind in ("no_action", "turn_incomplete"):
                     reply = NUDGE_MESSAGE
+                # Before restating: the fixture expects this question, so a lost_context verdict on it
+                # must not send the picks to a later questionnaire.
+                elif answerable and turn.clarifying_answers is not None and chat_result.clarifying_questions_item_id:
+                    built = build_clarifying_answers(chat_result, turn.clarifying_answers)
+                    matched_picks.update(built.matched_picks)
+                    skipped_questions = built.skipped
+                    pending_answers = (chat_result.clarifying_questions_item_id, built.answers)
+                    answers_sent = True
+                    reply = built.render()
                 elif record.lost_context is True and not any(r.lost_context is True for r in records):
                     # Once per turn, so a judge mistake cannot loop the user on restating.
                     reply = reply_restating(render_transcript(transcript[:-1]), response_text)
@@ -964,6 +1150,8 @@ def run_agentic_conversation(
                     reply = answers_left.pop(0)
                 else:
                     reply = reply_from_facts(render_transcript(transcript[:-1]), response_text, turn.set_answers)
+                if answerable and not answers_sent:
+                    answers_unsent = "the first reply's questionnaire has no itemId to send clarifying_answers to"
                 record.reply = clip(reply)
                 records.append(record)
 
@@ -988,6 +1176,67 @@ def run_agentic_conversation(
             if final_result and output_present:
                 output_correct, mismatches = _check_output_correct(resolved_turn, final_result, all_tool_calls)
 
+            # A turn with no complete reply has nothing to read; its chat error or missing output fails it.
+            clarification_correct: bool | None = None
+            clarification_problems: list[str] = []
+            clarification_ungraded: str | None = None
+            if turn.expects_clarification is not None and first_reply is not None:
+                checks_options = (
+                    turn.clarifying_options_grounded
+                    or turn.clarifying_option_titles is not None
+                    or turn.clarifying_option_objects is not None
+                )
+                workspace_objects: WorkspaceObjects | None = None
+                # Only a questionnaire with options has anything to check. The listing is kept for the run
+                # and read again once a turn has written to the workspace, since an option can name an
+                # object an earlier turn created.
+                if checks_options and first_reply.has_options:
+                    written = created.state()
+                    if workspace_listing is None or workspace_listing[0] != written:
+                        read_started = time.monotonic()
+                        try:
+                            workspace_listing = (written, fetch_workspace_objects(sdk, workspace_id))
+                        except Exception as exc:  # any failure leaves the options unchecked, never passed
+                            workspace_listing = None
+                            clarification_ungraded = f"workspace objects not readable, options left unchecked: {exc}"
+                            print(f"[CLARIFICATION] {clarification_ungraded}")
+                        workspace_read_s += time.monotonic() - read_started
+                    if workspace_listing is not None:
+                        workspace_objects = workspace_listing[1]
+                clarification_problems = clarification_mismatches(
+                    first_reply,
+                    expects_clarification=turn.expects_clarification,
+                    expected_form=turn.expected_clarification_form,
+                    max_questions=turn.max_clarifying_questions,
+                    objects=workspace_objects,
+                    grounded=turn.clarifying_options_grounded,
+                    titled_as=turn.clarifying_option_titles,
+                    candidates=turn.clarifying_option_objects,
+                    reads_questionnaire=turn.reads_questionnaire,
+                )
+                if turn.clarifying_answers is not None and not answers_sent:
+                    # Every check after the answers would pass with nothing to read.
+                    if answers_unsent is None:
+                        answers_unsent = (
+                            "the first reply asked no questionnaire to answer"
+                            if first_reply.form != "questionnaire"
+                            else "the turn ended before clarifying_answers were sent"
+                        )
+                    clarification_problems.append(f"clarifying_answers not sent: {answers_unsent}")
+                if turn.clarifying_answers is not None and answers_sent:
+                    clarification_problems += after_answers_mismatches(
+                        unmatched_picks=[p.label for p in turn.clarifying_answers if p.label not in matched_picks],
+                        skipped=skipped_questions,
+                        replies_after=replies_after_answers,
+                        states_skipped_assumption=turn.states_skipped_assumption,
+                        skipped_questions_not_reasked=turn.skipped_questions_not_reasked,
+                        expects_confirmation_after_answers=turn.expects_confirmation_after_answers,
+                    )
+                if clarification_problems:
+                    clarification_correct = False
+                elif clarification_ungraded is None:
+                    clarification_correct = True
+
             # Capture metric output for $ref resolution in subsequent turns.
             if final_result and turn.expected_output_type == "metric":
                 metric_data = _extract_metric_result(all_tool_calls)
@@ -1012,11 +1261,14 @@ def run_agentic_conversation(
                     mismatches=mismatches,
                     clarifications=records,
                     depends_on=list(turn.depends_on),
+                    clarification_correct=clarification_correct,
+                    clarification_mismatches=clarification_problems,
+                    clarification_ungraded=clarification_ungraded,
                 )
             )
 
         # Taken here, not after the finally: its cleanup is not part of the run.
-        run_latency_s = time.monotonic() - run_started - conversation_swap_s
+        run_latency_s = time.monotonic() - run_started - conversation_swap_s - workspace_read_s
     finally:
         if owns_conversation and conversation_id:
             client.delete_conversation(conversation_id)
@@ -1208,6 +1460,13 @@ def evaluate_agentic_conversation(
                         value=float(tr.skill_success),
                         data_type="BOOLEAN",
                     )
+                    if tr.clarification_correct is not None:
+                        ctx.score(
+                            tid,
+                            name=f"turn_{tr.turn_id}_clarification_correct",
+                            value=float(tr.clarification_correct),
+                            data_type="BOOLEAN",
+                        )
                 strict_checks = {
                     "conversation_success": result.conversation_success,
                     "full_skill_coverage": result.full_skill_coverage,

gooddata_eval/core/chat/sse_client.py

@@ -206,6 +206,7 @@ class _SseAccumulator:
     alert_proposals: list[dict[str, Any]] = field(default_factory=list)
     search_results: list[dict[str, Any]] = field(default_factory=list)
     unhandled_parts: list[dict[str, Any]] = field(default_factory=list)
+    clarifying_questions_item_id: str | None = None
     tool_call_events: list[dict[str, Any]] = field(default_factory=list)
     call_id_to_event_index: dict[str, int] = field(default_factory=dict)
     reasoning_steps: list[dict[str, Any]] = field(default_factory=list)
@@ -226,7 +227,8 @@ def _handle_text(content: dict[str, Any], acc: _SseAccumulator) -> None:
         acc.text_parts.append(text)
 
 
-def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator) -> None:
+def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator, item_id: str | None = None) -> None:
+    """Sort the item's parts; a ``clarifyingQuestions`` part keeps ``item_id``, since answers number questions per item."""
     for part in content.get("parts", []):
         ptype = part.get("type")
         if ptype == "text":
@@ -247,7 +249,8 @@ def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator) -> None:
         else:
             if ptype not in _KNOWN_PART_TYPES:
                 _log.warning("unknown multipart part type %r; captured as an unhandled part", ptype)
-            acc.unhandled_parts.append(part)
+            tagged = ptype == "clarifyingQuestions" and item_id
+            acc.unhandled_parts.append({**part, "itemId": item_id} if tagged else part)
 
 
 def _handle_reasoning(content: dict[str, Any], acc: _SseAccumulator) -> None:
@@ -300,6 +303,7 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
         "alertProposals": acc.alert_proposals,
         "searchResults": acc.search_results,
         "unhandledParts": acc.unhandled_parts,
+        "clarifyingQuestionsItemId": acc.clarifying_questions_item_id,
         "toolCallEvents": acc.tool_call_events,
         "reasoningStepCount": len(acc.reasoning_steps),
         "reasoningSteps": [step["summary"] for step in acc.reasoning_steps],
@@ -439,7 +443,9 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
             if ctype == "text":
                 _handle_text(content, acc)
             elif ctype == "multipart":
-                _handle_multipart(content, acc)
+                _handle_multipart(content, acc, item_id=item.get("itemId"))
+                if any(p.get("type") == "clarifyingQuestions" for p in content.get("parts", [])):
+                    acc.clarifying_questions_item_id = item.get("itemId") or acc.clarifying_questions_item_id
             elif ctype == "reasoning":
                 _handle_reasoning(content, acc)
             elif ctype == "toolCall":
@@ -540,11 +546,31 @@ class ChatClient:
 
     def send_message(
         self, conversation_id: str, question: str, *, user_context: dict[str, Any] | None = None
+    ) -> ChatResult:
+        return self._send(conversation_id, {"type": "text", "text": question}, user_context=user_context)
+
+    def send_clarifying_answers(
+        self,
+        conversation_id: str,
+        questions_item_id: str,
+        answers: list[dict[str, Any]],
+        *,
+        user_context: dict[str, Any] | None = None,
+    ) -> ChatResult:
+        """Answer a questionnaire the way the chat panel does: positions plus the texts shown.
+
+        A question left out of ``answers`` is skipped; an empty list skips them all.
+        """
+        content = {"type": "clarifyingAnswers", "questionsItemId": questions_item_id, "answers": answers}
+        return self._send(conversation_id, content, user_context=user_context)
+
+    def _send(
+        self, conversation_id: str, content: dict[str, Any], *, user_context: dict[str, Any] | None = None
     ) -> ChatResult:
         url = f"{self._base}/{conversation_id}/messages"
         join_headers, joined = self._join_item_trace(conversation_id)
         headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json", **join_headers}
-        body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
+        body: dict[str, Any] = {"item": {"role": "user", "content": content}}
         options: dict[str, str] = {}
         if self._reasoning_effort is not None:
             options["reasoningEffort"] = self._reasoning_effort

gooddata_eval/core/models.py

@@ -351,6 +351,9 @@ class ChatResult(BaseModel):
     alert_proposals: list[dict] = Field(default_factory=list, alias="alertProposals")
     search_results: list[dict] = Field(default_factory=list, alias="searchResults")
     unhandled_parts: list[dict] = Field(default_factory=list, alias="unhandledParts")
+    # The itemId of the assistant item holding a clarifyingQuestions part: a clarifyingAnswers
+    # message names it as questionsItemId.
+    clarifying_questions_item_id: str | None = Field(default=None, alias="clarifyingQuestionsItemId")
     tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents")
     reasoning_step_count: int = Field(default=0, alias="reasoningStepCount")
     reasoning_steps: list[str] = Field(default_factory=list, alias="reasoningSteps")

{gooddata_eval-1.79.1.dev1.dist-info → gooddata_eval-1.79.2.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.79.1.dev1
+Version: 1.79.2.dev1
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.79.1.dev1
+Requires-Dist: gooddata-sdk~=1.79.2.dev1
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15

{gooddata_eval-1.79.1.dev1.dist-info → gooddata_eval-1.79.2.dev1.dist-info}/RECORD

@@ -10,7 +10,7 @@ gooddata_eval/core/config.py,sha256=hla5t294uHC5Q9bzrhPjsdkwEcetLsEH-GqGAhs94b4,
 gooddata_eval/core/connection.py,sha256=mh-Do5-Ti0-DXXikfCIH10eTTUbKuZUAS_sLoPNDJ5I,1229
 gooddata_eval/core/granularity.py,sha256=CbniwgxtVj0oyf41UjeeZbBzOpeisSBlAfdkL79e6O0,3932
 gooddata_eval/core/interaction.py,sha256=PUaJ1zdMwXBKjoNXNFXVgt_b8_i0LWdbMZndEQ3F6UM,2093
-gooddata_eval/core/models.py,sha256=uYKDfGUqVFfCzEDPHwjlJNvBjjataLBy-W0oMprx9e8,25458
+gooddata_eval/core/models.py,sha256=Dzpdq7jkPJ2foDg7Wjx__S8emNPyKVihoDFLpwi5iQ4,25698
 gooddata_eval/core/number_format.py,sha256=bQv3YqlMtBHNSMZx2_qGMWXFDdeXNkAGVg3WFIKUd-k,9264
 gooddata_eval/core/runner.py,sha256=AXgx3MJsFs1CKFogggJwa6Lrt4owRPjQVCr6pt1WJHo,21022
 gooddata_eval/core/scoring.py,sha256=buY7ZoO08Dt59m2PPhFeNOyix5yYlf4mVxFyOn-aibk,23778
@@ -18,14 +18,15 @@ gooddata_eval/core/timing.py,sha256=1grXpr48OH4GcAmeobhjwnRWqoMJwcjj-LLpHdS_raY,
 gooddata_eval/core/workspace.py,sha256=S8exiy5gpDtcRtT8-i2fIRALF34Pbo0J46xm8X_ldus,11846
 gooddata_eval/core/agentic/__init__.py,sha256=i2g22ousTYXi5FZeN5X0WVpa67RQXThwli8KcYfIYxM,4432
 gooddata_eval/core/agentic/_catalog.py,sha256=My4SP5DteSJlp2WOQ66iemJkX3ANmdqrsgANDEm6u10,4194
-gooddata_eval/core/agentic/_conversation_context.py,sha256=43ryGOQ9kTLLrzRxBphUn_87290vTZQ6oqWLnzjKYbA,16270
+gooddata_eval/core/agentic/_clarification.py,sha256=X3XZrF-Y1GsvYpv3UacBeQw-vzIsXBlNjMYWEeyyLJA,22651
+gooddata_eval/core/agentic/_conversation_context.py,sha256=vh5mUgRs4h-tn2039dn47xkQfa3s-RXwDWZW_UeXAas,17175
 gooddata_eval/core/agentic/_failed_runs.py,sha256=iXN80ORMrIKEhV1QKGbyEZPP2w__mLPSrkm3hh-dTwM,3048
 gooddata_eval/core/agentic/_gate.py,sha256=irI4eWOCfmsbKd6l-HdpoAlnWkq2wjo4b5crGU_Er1o,2631
 gooddata_eval/core/agentic/_langfuse.py,sha256=NWhPgpd2NkYmAPBiT8VCOd_cddhhsmDGmcWmsvyGLGs,29729
 gooddata_eval/core/agentic/_trace_linker.py,sha256=mSY_7UDZU97r5qcaDxMoyPQ46944XzbzFbMRd74G87k,15760
 gooddata_eval/core/agentic/alert_skill.py,sha256=Me7YrAAGWG6iqzJR18JdhVsri4LG4J1itIUdzgboZU0,60105
 gooddata_eval/core/agentic/anomaly_detection.py,sha256=H_rKx6fCX3S11HzGAQGhy092YMmGj4foFBmrx6U-LHE,27701
-gooddata_eval/core/agentic/conversation.py,sha256=_OLlST1twnSPnOQGfuuFCFKu92uwLVh2YBBA_nOW-5w,60004
+gooddata_eval/core/agentic/conversation.py,sha256=-qHSwIsrBqQOlYNe1GcMSGGb3HGZqdXG3dSLNglC8oo,75510
 gooddata_eval/core/agentic/dashboard_skill.py,sha256=9ts12qLwxYQUZLYmUhDlxbxCkIiTgr4JdyElF2-J3dA,98041
 gooddata_eval/core/agentic/document_skill.py,sha256=Uukqc6voa3lXbcSVgfzQyPH_kEFu3jrbrN9EDIMIuYA,45231
 gooddata_eval/core/agentic/general_question.py,sha256=pOVYlJF9eKZwTqdPVsrH4X1OC-tXI7CMCFDdPNdBIMw,17758
@@ -38,7 +39,7 @@ gooddata_eval/core/agentic/visualization.py,sha256=o4RVS-nRCVbVXjbE_jHuGzgW8x3hR
 gooddata_eval/core/agentic/what_if.py,sha256=v5u9Yl45QGpVD2IHvPhVW_XOixI10IK4N-5_zP9Yymw,26365
 gooddata_eval/core/chat/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/chat/render.py,sha256=CqsKN8__DlaiJj_bmcPZ1poc5LfUqtz7EzKRDbks4gU,2082
-gooddata_eval/core/chat/sse_client.py,sha256=yGUbWg7gr7ycVfEpvpmGUCIQ38nW0jiMmuJB9_va4UY,31921
+gooddata_eval/core/chat/sse_client.py,sha256=fIWOmw3KEChiwmzS4I21Y_b7VXzDecqoGLbJUWfbOkU,33373
 gooddata_eval/core/dataset/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/dataset/from_insights.py,sha256=2eYh_84070t2BMeW2XPLAwzrJs7vKbFYMeCppvkDiEo,40880
 gooddata_eval/core/dataset/langfuse_source.py,sha256=II-6fenEPhdyfckrL0iVdJpavjv67wUvy7OIuvHntrg,8697
@@ -77,8 +78,8 @@ gooddata_eval/core/summary/http_client.py,sha256=Umae42fiiJQ0e4hv63NRROHYvcFAGgX
 gooddata_eval/core/takeaway/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/takeaway/evaluate.py,sha256=ei7DLSlG_Ove_R551Ofdw8WISf178XknU58jBWxB_is,11585
 gooddata_eval/core/takeaway/http_client.py,sha256=yc3FNPArjfcSdAE7naRf7g_aW7BbJhwsfSEvAFd67Ik,4141
-gooddata_eval-1.79.1.dev1.dist-info/METADATA,sha256=9_msqZ1neRnpLckeQYdvR85BjuB6sXpdCe09WSgWzdE,60351
-gooddata_eval-1.79.1.dev1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
-gooddata_eval-1.79.1.dev1.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
-gooddata_eval-1.79.1.dev1.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
-gooddata_eval-1.79.1.dev1.dist-info/RECORD,,
+gooddata_eval-1.79.2.dev1.dist-info/METADATA,sha256=ghQfKEWHtHeBM_0gSg6YXuPJzGtnoPiXkYZ_g_pdSe8,60351
+gooddata_eval-1.79.2.dev1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+gooddata_eval-1.79.2.dev1.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
+gooddata_eval-1.79.2.dev1.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
+gooddata_eval-1.79.2.dev1.dist-info/RECORD,,