gooddata-eval 1.74.1.dev1__py3-none-any.whl → 1.74.1.dev2__py3-none-any.whl

gooddata_eval/core/agentic/_catalog.py

@@ -2,6 +2,41 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+from enum import Enum
+
+
+class AnomalyDetectionGranularity(str, Enum):
+    """Detection intervals an anomaly alert accepts.
+
+    Mirrors gen-ai's enum of the same name; `StrEnum` is unavailable on the 3.10 floor, so
+    the `str` mixin carries the comparison against the raw tool argument.
+    """
+
+    HOUR = "HOUR"
+    DAY = "DAY"
+    WEEK = "WEEK"
+    MONTH = "MONTH"
+    QUARTER = "QUARTER"
+    YEAR = "YEAR"
+
+    @classmethod
+    def parse(cls, value: object) -> AnomalyDetectionGranularity | None:
+        """Coerce a fixture value, or None when the fixture states none.
+
+        Raises ValueError on an unknown interval: fixtures are hand-written, and a typo has
+        to fail before the run spends an API call rather than score the item against an
+        interval the product cannot produce.
+        """
+        if value is None:
+            return None
+        candidate = str(value).strip().upper()
+        if not candidate:
+            return None
+        try:
+            return cls(candidate)
+        except ValueError:
+            expected = ", ".join(member.value for member in cls)
+            raise ValueError(f"Invalid granularity {value!r}; expected one of {expected}.") from None
 
 
 @dataclass
@@ -28,6 +63,10 @@ class CatalogMetricAlert:
     """List of recipient email addresses."""
     filters: list | str | None = None
     """Attribute filters applied to the alert condition."""
+    attributes: list | None = None
+    """Expected group-by attributes; ``None`` means the fixture states no expectation."""
+    granularity: AnomalyDetectionGranularity | None = None
+    """Detection interval for an ANOMALY alert (DAY/WEEK/MONTH/...). Not a date filter."""
 
     @classmethod
     def from_dict(cls, d: dict) -> CatalogMetricAlert:
@@ -46,4 +85,6 @@ class CatalogMetricAlert:
             metric_id=d.get("metric_id"),
             recipients=recipients,
             filters=d.get("filters"),
+            attributes=d.get("attributes"),
+            granularity=AnomalyDetectionGranularity.parse(d.get("granularity")),
         )

gooddata_eval/core/agentic/alert_skill.py

@@ -11,7 +11,7 @@ from typing import Any
 
 from gooddata_sdk import GoodDataSdk
 
-from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
+from gooddata_eval.core.agentic._catalog import AnomalyDetectionGranularity, CatalogMetricAlert
 from gooddata_eval.core.agentic._trace_linker import (
     RunIdentity,
     RunTraceContext,
@@ -21,6 +21,7 @@ from gooddata_eval.core.agentic._trace_linker import (
     submit_trace_scoring,
     utc_now,
 )
+from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import (
@@ -29,6 +30,7 @@ from gooddata_eval.core.models import (
     ReasoningStepEvent,
     ToolCallEvent,
     build_latency_breakdown,
+    shift_and_index_events,
 )
 
 try:
@@ -126,6 +128,94 @@ def _check_filters(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     return _deep_subset(exp_filters, act_filters)
 
 
+def _attribute_label_ids(items: list, *, side: str) -> list[str]:
+    """Canonicalise group-by entries to bare label ids, whatever spelling they arrive in.
+
+    The two sides of the comparison speak different vocabularies for the same grouping.
+    Fixtures author the AAC tool-input form, ``{"using": "label/x"}``; ``create_metric_alert``
+    receives the resolved AFM form, ``{"localIdentifier": "a0", "label": {"identifier":
+    {"id": "x", "type": "label"}}}``, forwarded verbatim from ``prepare_metric_alert_proposal``.
+    Identity is therefore the only thing they can be compared on.
+
+    A shape not listed here, or a URI prefix other than ``label/``, raises: ``label/x`` and
+    ``attribute/x`` are different objects, and an unknown spelling must fail loudly rather
+    than quietly compare unequal.
+    """
+    if not isinstance(items, list):
+        raise ValueError(f"Unrecognised {side} group-by attributes, expected a list: {items!r}")
+    ids: list[str] = []
+    for item in items:
+        raw: object = None
+        if isinstance(item, str):
+            raw = item
+        elif isinstance(item, dict):
+            label = item.get("label")
+            identifier = item.get("identifier")
+            if isinstance(item.get("using"), str):
+                raw = item["using"]
+            elif isinstance(label, dict) and isinstance(label.get("identifier"), dict):
+                raw = label["identifier"].get("id")
+            elif isinstance(identifier, dict):
+                raw = identifier.get("id")
+        if not isinstance(raw, str) or not raw:
+            raise ValueError(f"Unrecognised {side} group-by attribute entry: {item!r}")
+        prefix, slash, rest = raw.partition("/")
+        if not slash:
+            ids.append(raw)
+        elif prefix == "label" and rest:
+            ids.append(rest)
+        else:
+            raise ValueError(f"Unrecognised {side} group-by attribute reference: {raw!r}")
+    return ids
+
+
+def _check_attributes(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+    """Compare group-by identity only.
+
+    Per-entry properties — ``showAllValues``, the converter-assigned ``localIdentifier`` —
+    are deliberately not asserted, and the comparison is a multiset so entry order does not
+    matter.
+    """
+    exp_attributes = expected.attributes
+    if exp_attributes is None:
+        return True
+    act_attributes = actual_args.get("attributes")
+    if act_attributes is None:
+        # Arguments are raw `json.loads` output, where an unset nullable argument arrives as
+        # null rather than absent. Both spellings of "no grouping" have to land on [], which
+        # is why this is not `actual_args.get("attributes", [])`.
+        act_attributes = []
+    elif not isinstance(act_attributes, list):
+        # An argument that is not a list of groupings is the agent answering wrongly, so it
+        # scores False. Raising instead would make the runner record an ERROR, and errored
+        # items are excluded from the failure count — a malformed answer must not rank above
+        # a merely wrong one. An unreadable *entry* still raises, in `_attribute_label_ids`:
+        # entries are typed at the tool boundary, so the plausible cause there is the wire
+        # format moving, which has to be unmissable.
+        return False
+    if not exp_attributes:
+        return not act_attributes
+    exp_ids = sorted(_attribute_label_ids(exp_attributes, side="expected"))
+    act_ids = sorted(_attribute_label_ids(act_attributes, side="actual"))
+    return exp_ids == act_ids
+
+
+def _check_granularity(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+    """Compare the ANOMALY detection interval when the fixture states one.
+
+    ``None`` means unasserted, mirroring ``attributes``: only the ANOMALY items carry a
+    ``Granularity``, and every other item must stay unaffected. The expectation is already
+    canonical by the time it lands here; the tool argument is a raw string, so only that
+    side needs folding.
+    """
+    if expected.granularity is None:
+        return True
+    actual = actual_args.get("granularity")
+    if not actual:
+        return False
+    return str(actual).strip().upper() == expected.granularity.value
+
+
 def _check_metric(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     if not expected.metric_id:
         return True
@@ -257,20 +347,38 @@ def generate_simulated_alert_response(
         )
     elif filters == []:
         filters_rule = (
-            "5. Your alert must have NO filters and NO date/time window — it evaluates over all time. "
-            "If the agent asks which time period each check should cover, or offers a choice such as "
-            "'last Day / Week / Month', do NOT pick one: reply that you want no date filter at all, "
-            "all time. Never invent a period, a granularity or an 'evaluate each run on a X basis' "
-            "instruction the goal did not ask for.\n"
+            "5. Your alert must have NO filters and NO date/time window on the metric — it evaluates "
+            "over all time. If the agent asks which time period each check should cover, or offers a "
+            "choice such as 'last Day / Week / Month', do NOT pick one: reply that you want no date "
+            "filter at all, all time.\n"
         )
     else:
         filters_rule = (
             "5. Ask only for the filters your original request implies — do not invent an evaluation "
-            "period, granularity or date window that was not requested. If the agent offers a choice "
+            "period or date window that was not requested. If the agent offers a choice "
             "such as 'last Day / Week / Month' that your request never mentioned, say you do not want "
             "a date window.\n"
         )
 
+    if operator == "ANOMALY":
+        # The fallback keeps the conversation alive when the fixture names no interval -- an
+        # anomaly alert cannot be created without one. It is deliberately NOT mirrored into
+        # `expected.granularity`: `_check_granularity` asserts only what the fixture stated,
+        # and scoring an item against an interval it never asked for is the defect this rule
+        # exists to undo.
+        granularity = (expected.granularity or AnomalyDetectionGranularity.DAY).value
+        anomaly_rule = (
+            "7. This is an ANOMALY alert. Anomaly detection REQUIRES a time granularity, and that "
+            f"granularity is NOT a date filter. State it in your first reply and repeat it whenever "
+            f"asked: use {granularity} granularity. Rule 5 constrains filters on the metric only — it "
+            "never applies to this detection interval, so never refuse to give one.\n"
+        )
+    else:
+        anomaly_rule = (
+            "7. Do not invent an evaluation period, a granularity or an 'evaluate each run on a X "
+            "basis' instruction your goal never asked for.\n"
+        )
+
     original_request = f'Your original request to the agent was: "{question}"\n' if question else ""
 
     system_prompt = (
@@ -294,8 +402,7 @@ def generate_simulated_alert_response(
         "   Do not wait for the agent to ask — state it alongside the metric and condition answers.\n"
         + filters_rule
         + f"6. Proactively state how often you want to be alerted in your first reply: {trigger_request}. "
-        "   Repeat it if the agent proposes a different cadence.\n"
-        "Reply concisely and directly."
+        "   Repeat it if the agent proposes a different cadence.\n" + anomaly_rule + "Reply concisely and directly."
     )
 
     messages: list = [{"role": "system", "content": system_prompt}]
@@ -334,6 +441,8 @@ class AlertEvaluation:
     filters_correct: bool
     metric_correct: bool
     recipients_correct: bool
+    attributes_correct: bool = True
+    granularity_correct: bool = True
 
     @property
     def strict_pass(self) -> bool:
@@ -346,6 +455,8 @@ class AlertEvaluation:
                 self.filters_correct,
                 self.metric_correct,
                 self.recipients_correct,
+                self.attributes_correct,
+                self.granularity_correct,
             ]
         )
 
@@ -409,6 +520,35 @@ def _normalize_expected_filters(expected: dict) -> list | str | None:
     return None
 
 
+_NO_GROUPING_MARKERS = ("none", "no grouping")
+
+
+def _normalize_expected_attributes(expected: dict) -> list | None:
+    """
+    * ``Attributes`` list        -> that list (exact expectation)
+    * "None" / "no grouping"     -> ``[]``   (stated: no group-by; extras fail)
+    * absent, or other prose     -> ``None`` (unstated; grouping not asserted)
+
+    A date narrows an alert as a group-by as well as a filter, and a group-by makes it fire
+    per period value instead of on the latest one — so ``[]`` has to be expressible separately
+    from "absent", exactly as it is for ``filters``.
+
+    The simulated user is told nothing about groupings, so a non-empty expectation requires the
+    item's own question to request that grouping; ``[]`` needs no such support, because the
+    simulated user does not invent a grouping and the check verifies it did not.
+    """
+    attributes = _case_insensitive_get(expected, "attributes")
+    if isinstance(attributes, list):
+        # Validated here so a malformed fixture fails before the run spends an API call.
+        _attribute_label_ids(attributes, side="expected")
+        return attributes
+    if attributes is None:
+        return None
+    if isinstance(attributes, str):
+        return [] if any(kw in attributes.lower() for kw in _NO_GROUPING_MARKERS) else None
+    raise ValueError(f"Attributes expectation must be a list or a display string, got {type(attributes).__name__}")
+
+
 def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
     """Parse expected_output dict into CatalogMetricAlert, accepting display-format or internal-format keys."""
     operator = _case_insensitive_get(expected, "operator") or "GREATER_THAN"
@@ -433,6 +573,11 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
         recipients = list(raw_recip)
 
     filters = _normalize_expected_filters(expected)
+    attributes = _normalize_expected_attributes(expected)
+
+    granularity = AnomalyDetectionGranularity.parse(
+        _case_insensitive_get(expected, "granularity", "detection granularity")
+    )
 
     return CatalogMetricAlert(
         operator=operator,
@@ -443,6 +588,8 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
         metric_id=metric_id,
         recipients=recipients,
         filters=filters,
+        attributes=attributes,
+        granularity=granularity,
     )
 
 
@@ -526,21 +673,14 @@ def run_agentic_alert_skill(
                 chat_result = client.send_message(conv_id, current_question)
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
-                for tc in chat_result.tool_call_events or []:
-                    if tc.call_ts is not None:
-                        tc.call_ts += turn_offset
-                    if tc.result_ts is not None:
-                        tc.result_ts += turn_offset
-                    if tc.index is not None:
-                        tc.index += tool_index_offset
-                for rs in chat_result.reasoning_step_events or []:
-                    rs.ts += turn_offset
-                    rs.index += reasoning_index_offset
+                turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
+                    chat_result,
+                    turn_offset=turn_offset,
+                    tool_index_offset=tool_index_offset,
+                    reasoning_index_offset=reasoning_index_offset,
+                )
                 all_tool_call_events.extend(chat_result.tool_call_events or [])
                 all_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
-                tool_index_offset += len(chat_result.tool_call_events or [])
-                reasoning_index_offset += len(chat_result.reasoning_step_events or [])
-                turn_offset += chat_result.turn_wall_clock_sec or 0.0
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id
@@ -548,6 +688,8 @@ def run_agentic_alert_skill(
                 response_text = (chat_result.text_response or "").strip()
                 if not response_text and chat_result.alert_proposals:
                     response_text = render_alert_proposal(chat_result.alert_proposals[-1])
+                if not response_text:
+                    response_text = render_answer_text(chat_result)
                 # Stop if agent gave a completely empty response (stuck)
                 if not response_text and not chat_result.tool_call_events:
                     break
@@ -570,6 +712,8 @@ def run_agentic_alert_skill(
                 filters_correct=tool_called and _check_filters(expected, actual_args),
                 metric_correct=tool_called and _check_metric(expected, actual_args),
                 recipients_correct=tool_called and _check_recipients(expected, actual_args, sdk=sdk),
+                attributes_correct=tool_called and _check_attributes(expected, actual_args),
+                granularity_correct=tool_called and _check_granularity(expected, actual_args),
             )
             return AlertRunResult(
                 conversation_id=conv_id,
@@ -615,6 +759,8 @@ def run_agentic_alert_skill(
                 r.eval.filters_correct,
                 r.eval.metric_correct,
                 r.eval.recipients_correct,
+                r.eval.attributes_correct,
+                r.eval.granularity_correct,
             ]
         ),
     )
@@ -689,6 +835,8 @@ def evaluate_agentic_alert_skill(
                     "filters_correct": ev.filters_correct,
                     "metric_correct": ev.metric_correct,
                     "recipients_correct": ev.recipients_correct,
+                    "attributes_correct": ev.attributes_correct,
+                    "granularity_correct": ev.granularity_correct,
                 }
                 with ctx.observe(pt, run_idx) as tid:
                     for score_name, value in strict_checks.items():
@@ -735,6 +883,8 @@ def evaluate_agentic_alert_skill(
         "filters_correct": ev.filters_correct,
         "metric_correct": ev.metric_correct,
         "recipients_correct": ev.recipients_correct,
+        "attributes_correct": ev.attributes_correct,
+        "granularity_correct": ev.granularity_correct,
         "actual_alert_arguments": best.actual_alert_arguments,
         "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
     }
@@ -745,7 +895,9 @@ def evaluate_agentic_alert_skill(
             f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, "
             f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, "
             f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, "
-            f"recipients_correct={ev.recipients_correct}. "
+            f"recipients_correct={ev.recipients_correct}, "
+            f"attributes_correct={ev.attributes_correct}, "
+            f"granularity_correct={ev.granularity_correct}. "
             f"Actual args: {best.actual_alert_arguments}"
         )
         exc.reasoning_steps = best.reasoning_steps

gooddata_eval/core/agentic/conversation.py

@@ -6,10 +6,10 @@ from __future__ import annotations
 import json
 import re
 from dataclasses import dataclass, field
-from typing import Literal
+from typing import ClassVar, Literal
 
 from gooddata_sdk import GoodDataSdk
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from gooddata_eval.core.agentic._trace_linker import (
     RunIdentity,
@@ -22,6 +22,7 @@ from gooddata_eval.core.agentic._trace_linker import (
 )
 from gooddata_eval.core.agentic.alert_skill import render_alert_proposal
 from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids, _extract_metric_result
+from gooddata_eval.core.chat.render import render_answer_text
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import (
@@ -31,6 +32,7 @@ from gooddata_eval.core.models import (
     ReasoningStepEvent,
     ToolCallEvent,
     build_latency_breakdown,
+    shift_and_index_events,
 )
 from gooddata_eval.core.scoring import (
     check_filters,
@@ -65,7 +67,21 @@ class ConversationFixture(BaseModel):
 
 
 class TurnResult(BaseModel):
-    """Evaluation result for a single conversation turn."""
+    """Evaluation result for a single conversation turn.
+
+    The two skill fields measure DIFFERENT SCOPES, so a turn can legitimately report
+    ``skill_routing=True`` with an empty ``activated_skills``:
+
+    - ``activated_skills`` -- what THIS turn's own ``set_skills`` call declared. Empty
+      whenever the agent reused an already-active skill without re-declaring it.
+    - ``active_skills`` -- what was actually active DURING this turn: the last declared
+      set, carried over on turns that declare nothing. This is the set ``skill_routing``
+      is judged against, so a report never has to infer it.
+    - ``skill_routing`` -- whether ``expected_skill`` appears in ``active_skills``.
+
+    ``skill_routing=True`` with ``activated_skills=[]`` is the reused-skill case, not a
+    scoring bug -- ``active_skills`` shows where the credit came from.
+    """
 
     turn_id: str
     expected_skill: str
@@ -73,6 +89,8 @@ class TurnResult(BaseModel):
     output_present: bool
     no_error: bool
     activated_skills: list[str]
+    # Sorted for stable output: the source is a set, whose iteration order is not.
+    active_skills: list[str] = Field(default_factory=list)
     clarification_turns_used: int = 0
     output_correct: bool | None = None
 
@@ -80,6 +98,26 @@ class TurnResult(BaseModel):
     def skill_success(self) -> bool:
         return self.skill_routing and self.output_present and self.no_error
 
+    # Reported per turn in detail["turns"]. A name listed here that no longer exists on the
+    # model raises rather than silently emitting a stale key, which a hand-written dict
+    # literal of the same fields would not -- and model_dump deep-copies activated_skills,
+    # so a caller mutating the returned dict cannot reach back into this TurnResult.
+    _DETAIL_FIELDS: ClassVar[set[str]] = {
+        "turn_id",
+        "expected_skill",
+        "skill_routing",
+        "output_present",
+        "output_correct",
+        "activated_skills",
+        # What skill_routing was judged against -- without it, a turn showing
+        # skill_routing=True and activated_skills=[] looks like a scoring bug.
+        "active_skills",
+    }
+
+    def detail(self) -> dict:
+        """The subset of this result reported in detail["turns"] for one conversation turn."""
+        return self.model_dump(include=self._DETAIL_FIELDS)
+
 
 def _resolve_refs(
     expected_output: dict | None,
@@ -117,15 +155,42 @@ def _resolve_refs(
     return json.loads(resolved_raw)
 
 
-def _activated_skills(tool_call_events: list[ToolCallEvent]) -> list[str]:
-    """Collect all skill names passed to set_skills across all tool call events."""
-    skills: list[str] = []
+def _set_skills_declarations(tool_call_events: list[ToolCallEvent]) -> list[list[str]]:
+    """Every set_skills declaration in these events, in call order.
+
+    `skill_names` is the key the tool declares; `skills` is a legacy spelling kept as a
+    fallback. A call carrying neither is treated as declaring an empty list, which is what
+    the platform would do with one.
+    """
+    declarations: list[list[str]] = []
     for tc in tool_call_events:
         if tc.function_name != "set_skills":
             continue
         args = tc.parsed_arguments() or {}
-        skills.extend(args.get("skill_names") or args.get("skills") or [])
-    return list(set(skills))
+        names = args.get("skill_names")
+        if names is None:
+            names = args.get("skills")
+        declarations.append(list(names or []))
+    return declarations
+
+
+def _final_skill_declaration(tool_call_events: list[ToolCallEvent]) -> list[str] | None:
+    """The skill list from the LAST set_skills call, or None when there was no call.
+
+    set_skills replaces the active set, so when a turn issues several calls -- which it can,
+    since these events span every clarification sub-turn within one logical turn -- only the
+    final one describes the resulting state. Merging them would credit a skill that an
+    earlier call declared and a later one dropped.
+
+    An empty list is a real declaration: it deactivates everything. That has to stay
+    distinguishable from ``None`` ("no call at all"), which leaves the previous turn's set
+    untouched -- hence the Optional rather than just an empty list for both.
+
+    This answers "what is active NOW". For "was this skill ever exercised" -- what
+    full_skill_coverage asks -- use every declaration, not just the last one.
+    """
+    declarations = _set_skills_declarations(tool_call_events)
+    return declarations[-1] if declarations else None
 
 
 def _check_output_present(turn: TurnDefinition, chat_result: ChatResult) -> bool:
@@ -150,7 +215,7 @@ def _check_output_correct(turn: TurnDefinition, chat_result: ChatResult) -> bool
 
     Returns None when expected_output is absent (presence check only).
     """
-    from gooddata_eval.core.agentic.metric_skill import _normalize_maql  # noqa: PLC0415
+    from gooddata_eval.core.evaluators._maql import normalize_maql  # noqa: PLC0415
 
     otype = turn.expected_output_type
     expected = turn.expected_output
@@ -192,7 +257,7 @@ def _check_output_correct(turn: TurnDefinition, chat_result: ChatResult) -> bool
         metric_result = _extract_metric_result(chat_result.tool_call_events or [])
         if not metric_result:
             return False
-        return _normalize_maql(metric_result.get("maql", "")) == _normalize_maql(expected.get("maql", ""))
+        return normalize_maql(metric_result.get("maql", "")) == normalize_maql(expected.get("maql", ""))
 
     return None
 
@@ -310,6 +375,20 @@ def run_agentic_conversation(
     response_id: str | None = None
     conversation_tool_call_events: list[ToolCallEvent] = []
     conversation_reasoning_step_events: list[ReasoningStepEvent] = []
+    # The skills active right now, mirroring the platform's own state machine: set_skills
+    # REPLACES the active set rather than adding to it -- verified against the gen-ai
+    # service's skill registry, and stated in the tool's own description. So a turn that
+    # issues no set_skills call inherits the previous turn's set unchanged, while a turn
+    # that does issue one drops whatever it left out. Tracking this as a running UNION
+    # would credit a skill a later call had already switched off.
+    active_skills: set[str] = set()
+    # Every skill declared at any point, for full_skill_coverage. This asks a DIFFERENT
+    # question from active_skills -- "did the conversation ever exercise this skill" rather
+    # than "is it active now" -- so replace semantics do not apply: a skill switched on and
+    # later switched off was still exercised. Deriving coverage from the per-turn final
+    # declaration instead would drop any skill a turn declared and then replaced within
+    # itself (across its clarification sub-turns), a false FAIL on a genuine activation.
+    ever_declared_skills: set[str] = set()
     # Every send_message() call (across every logical turn AND every clarification
     # sub-turn within it) restarts call_ts/ts near 0 -- these run across the whole
     # conversation, not reset per logical turn, so every one of those calls shifts them.
@@ -337,6 +416,10 @@ def run_agentic_conversation(
                         output_present=False,
                         no_error=False,
                         activated_skills=[],
+                        # The turn never ran, so it declared nothing -- but a set carried over
+                        # from an earlier turn is still active, and reporting [] here would
+                        # read as "nothing was active", which is a different claim.
+                        active_skills=sorted(active_skills),
                         output_correct=False,
                     )
                 )
@@ -351,22 +434,15 @@ def run_agentic_conversation(
             for _iter in range(max_clarification_turns + 1):
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
-                for tc in chat_result.tool_call_events or []:
-                    if tc.call_ts is not None:
-                        tc.call_ts += turn_offset
-                    if tc.result_ts is not None:
-                        tc.result_ts += turn_offset
-                    if tc.index is not None:
-                        tc.index += tool_index_offset
-                for rs in chat_result.reasoning_step_events or []:
-                    rs.ts += turn_offset
-                    rs.index += reasoning_index_offset
+                turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
+                    chat_result,
+                    turn_offset=turn_offset,
+                    tool_index_offset=tool_index_offset,
+                    reasoning_index_offset=reasoning_index_offset,
+                )
                 all_tool_calls.extend(chat_result.tool_call_events or [])
                 conversation_tool_call_events.extend(chat_result.tool_call_events or [])
                 conversation_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
-                tool_index_offset += len(chat_result.tool_call_events or [])
-                reasoning_index_offset += len(chat_result.reasoning_step_events or [])
-                turn_offset += chat_result.turn_wall_clock_sec or 0.0
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
 
@@ -376,6 +452,8 @@ def run_agentic_conversation(
                 response_text = (chat_result.text_response or "").strip()
                 if not response_text and chat_result.alert_proposals:
                     response_text = render_alert_proposal(chat_result.alert_proposals[-1])
+                if not response_text:
+                    response_text = render_answer_text(chat_result)
                 if not response_text and not chat_result.tool_call_events:
                     break
                 if clarification_turns >= max_clarification_turns:
@@ -384,8 +462,17 @@ def run_agentic_conversation(
                 total_clarification_turns += 1
                 current_message = _get_sim_user_response(response_text, resolved_turn, resolved_expected)
 
-            activated = _activated_skills(all_tool_calls)
-            skill_routing = turn.expected_skill in activated if activated else False
+            # `declared` is what THIS turn's final set_skills call asked for (None when it
+            # made no call); `active_skills` is what is actually active during the turn. No
+            # call carries the previous set over; a call replaces it outright, including
+            # when it declares an empty list. See active_skills' declaration above.
+            declarations = _set_skills_declarations(all_tool_calls)
+            for declaration in declarations:
+                ever_declared_skills.update(declaration)
+            declared = declarations[-1] if declarations else None
+            if declared is not None:
+                active_skills = set(declared)
+            skill_routing = turn.expected_skill in active_skills
             output_present = _check_output_present(resolved_turn, final_result) if final_result else False
             output_correct = (
                 _check_output_correct(resolved_turn, final_result) if (final_result and output_present) else None
@@ -409,7 +496,8 @@ def run_agentic_conversation(
                     skill_routing=skill_routing,
                     output_present=output_present,
                     no_error=True,  # SDK raises on errors; reaching here means no critical error.
-                    activated_skills=activated,
+                    activated_skills=declared or [],
+                    active_skills=sorted(active_skills),
                     clarification_turns_used=clarification_turns,
                     output_correct=output_correct,
                 )
@@ -422,8 +510,10 @@ def run_agentic_conversation(
             _delete_metric(sdk, workspace_id, metric_id)
         client.close()
 
-    activated_all = {skill for tr in turn_results for skill in tr.activated_skills}
-    full_skill_coverage = set(fixture.expected_skills).issubset(activated_all)
+    # Not derived from TurnResult.activated_skills: that field carries only each turn's FINAL
+    # declaration, so a skill replaced within its own turn is absent from it despite having
+    # been activated. See ever_declared_skills' declaration above.
+    full_skill_coverage = set(fixture.expected_skills).issubset(ever_declared_skills)
     conversation_success = all(tr.skill_success for tr in turn_results)
 
     return ConversationResult(
@@ -443,17 +533,7 @@ def _conversation_detail(result: ConversationResult) -> dict:
     return {
         "full_skill_coverage": result.full_skill_coverage,
         "total_clarification_turns": result.total_clarification_turns,
-        "turns": [
-            {
-                "turn_id": tr.turn_id,
-                "expected_skill": tr.expected_skill,
-                "skill_routing": tr.skill_routing,
-                "output_present": tr.output_present,
-                "output_correct": tr.output_correct,
-                "activated_skills": tr.activated_skills,
-            }
-            for tr in result.turn_results
-        ],
+        "turns": [tr.detail() for tr in result.turn_results],
         "latency_breakdown": build_latency_breakdown(result.tool_call_events, result.reasoning_step_events),
     }