graph-agents-cli 0.3.1__py3-none-any.whl

graph_agents_cli/eval/gate.py

@@ -0,0 +1,420 @@
+# Copyright 2026 graph-agents-cli contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The evaluation gate: case statuses, quality rates, exit codes.
+
+One rule: complete case accounting, every deterministic check and every
+mandatory judge metric must pass. Only judge metrics listed under
+``quality_metrics:`` may miss their per-case threshold at a rate bounded by
+``min_pass_rate``. Statuses: ``passed``, ``failed``, ``quality_below_threshold``,
+``error``, ``missing``. Exit codes: 0 gate met; 1 any failed or a quality
+metric under its rate; 2 any error or missing; 3 configuration error.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from rich.markup import escape
+from rich.table import Table
+
+from graph_agents_cli._output import Console
+from graph_agents_cli.eval._common import (
+    EXIT_GATE_FAILED,
+    EXIT_INCOMPLETE,
+    EXIT_OK,
+    EvalConfigError,
+)
+from graph_agents_cli.eval.checks import run_checks
+from graph_agents_cli.eval.config import EvalConfig, render_judge_prompt, resolve_threshold
+from graph_agents_cli.eval.dataset import SCOPE_ALL_TURNS, EvalCase
+from graph_agents_cli.eval.transcript import RenderedCase, render_case
+
+STATUS_PASSED = "passed"
+STATUS_FAILED = "failed"
+STATUS_QUALITY = "quality_below_threshold"
+STATUS_ERROR = "error"
+STATUS_MISSING = "missing"
+STATUSES: tuple[str, ...] = (
+    STATUS_PASSED,
+    STATUS_FAILED,
+    STATUS_QUALITY,
+    STATUS_ERROR,
+    STATUS_MISSING,
+)
+# Higher is worse; used by ``eval compare`` to detect regressions.
+STATUS_RANK: dict[str, int] = {
+    STATUS_PASSED: 0,
+    STATUS_QUALITY: 1,
+    STATUS_FAILED: 2,
+    STATUS_ERROR: 3,
+    STATUS_MISSING: 3,
+}
+
+_STATUS_STYLE = {
+    STATUS_PASSED: "green",
+    STATUS_QUALITY: "yellow",
+    STATUS_FAILED: "red",
+    STATUS_ERROR: "red",
+    STATUS_MISSING: "magenta",
+}
+
+
+@dataclass
+class CaseGrade:
+    id: str
+    checks: dict[str, dict[str, Any]] = field(default_factory=dict)
+    judge_scores: dict[str, dict[str, Any]] = field(default_factory=dict)
+    missing: list[str] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+    failures: list[str] = field(default_factory=list)
+    quality_misses: list[str] = field(default_factory=list)
+    # What the judges of this case could not see in full (cut tool results,
+    # unrecorded earlier turns); reported, never silent.
+    judge_notes: list[str] = field(default_factory=list)
+    truncated_tool_results: int = 0
+    unrecorded_turns: int = 0
+    # `expect.scope: all_turns` on a multi-turn case whose trace has no per-turn
+    # records: its checks could read the final turn only.
+    checks_final_turn_only: bool = False
+
+    @property
+    def status(self) -> str:
+        if self.missing:
+            return STATUS_MISSING
+        if self.errors:
+            return STATUS_ERROR
+        if self.failures:
+            return STATUS_FAILED
+        if self.quality_misses:
+            return STATUS_QUALITY
+        return STATUS_PASSED
+
+    @property
+    def reasons(self) -> list[str]:
+        return [*self.missing, *self.errors, *self.failures, *self.quality_misses]
+
+    @property
+    def judgeable(self) -> bool:
+        """Judges run only for cases whose mandatory items have not already failed."""
+        return not (self.missing or self.errors or self.failures)
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "id": self.id,
+            "status": self.status,
+            "reasons": self.reasons,
+            "checks": self.checks,
+            "judge_scores": self.judge_scores,
+        }
+        if self.judge_notes:
+            data["judge_notes"] = list(self.judge_notes)
+        return data
+
+
+# --- deterministic stage ------------------------------------------------------
+
+
+def grade_deterministic(case: EvalCase, trace: dict[str, Any] | None) -> CaseGrade:
+    """Case accounting plus the deterministic checks the case declares."""
+    grade = CaseGrade(id=case.id)
+    if trace is None:
+        grade.missing.append("missing: no trace for case")
+        return grade
+    if trace.get("status") == "missing":
+        grade.missing.append(f"missing: {trace.get('error') or 'trace has no response'}")
+        return grade
+    if trace.get("status") == "error":
+        grade.errors.append(f"error: {trace.get('error') or 'generation failed'}")
+        return grade
+    if trace.get("response") is None:
+        grade.missing.append("missing: trace has no response")
+        return grade
+    grade.checks = run_checks(case.expect, trace)
+    turns = trace.get("turns")
+    grade.checks_final_turn_only = (
+        case.expect.get("scope") == SCOPE_ALL_TURNS
+        and len(case.user_messages()) > 1
+        and not (isinstance(turns, list) and turns)
+    )
+    for name, result in grade.checks.items():
+        if not result["passed"]:
+            grade.failures.append(f"{name}: {result['reason']}")
+    return grade
+
+
+# --- judge stage --------------------------------------------------------------
+
+
+def case_metrics(config: EvalConfig, case: EvalCase) -> dict[str, dict[str, Any]]:
+    """Metric name -> case-level spec for every judge/custom metric that applies."""
+    metrics: dict[str, dict[str, Any]] = {name: dict(spec) for name, spec in case.judge.items()}
+    for name in config.custom_metrics:
+        metrics.setdefault(name, {})
+    return metrics
+
+
+def validate_case_metrics(config: EvalConfig, case: EvalCase) -> None:
+    """Unknown metric or missing threshold is a configuration error (exit 3)."""
+    known = config.metric_names()
+    for name, spec in case_metrics(config, case).items():
+        if name not in known:
+            raise EvalConfigError(
+                f"case {case.id!r} declares unknown judge metric {name!r} "
+                f"(known: {', '.join(sorted(known))})"
+            )
+        resolve_threshold(config, name, spec)
+
+
+def item_id(case_id: str, metric: str) -> str:
+    return f"{case_id}/{metric}"
+
+
+def plan_judge_items(
+    config: EvalConfig, case: EvalCase, trace: dict[str, Any], grade: CaseGrade
+) -> list[dict[str, Any]]:
+    """Runner items for the judge and custom metrics of one judgeable case."""
+    if not grade.judgeable:
+        return []
+    items: list[dict[str, Any]] = []
+    response = trace.get("response")
+    response = "" if response is None else str(response)
+    rendered: RenderedCase | None = None
+    for name, spec in case_metrics(config, case).items():
+        threshold = resolve_threshold(config, name, spec)
+        quality = config.is_quality(name)
+        custom = config.custom_metrics.get(name)
+        grade.judge_scores[name] = {
+            "score": None,
+            "threshold": threshold,
+            "passed": None,
+            "quality": quality,
+            "reasoning": "",
+            "error": None,
+            # "judge" (an LLM judge) or "custom" (a project callable): errors say which.
+            "kind": "custom" if custom is not None else "judge",
+        }
+        if custom is not None:
+            items.append(
+                {
+                    "id": item_id(case.id, name),
+                    "kind": "custom",
+                    "case_id": case.id,
+                    "metric": name,
+                    "callable": custom.callable,
+                    "case": case.raw,
+                    "trace": trace,
+                }
+            )
+            continue
+        judge = config.judges[name]
+        if rendered is None:
+            # Judges see every turn (earlier replies and tool results), not only
+            # the user messages and the final reply.
+            rendered = render_case(case, trace, max_tool_result_chars=config.max_tool_result_chars)
+            grade.judge_notes = rendered.notes
+            grade.truncated_tool_results = rendered.truncated_results
+            grade.unrecorded_turns = rendered.unrecorded_turns
+        items.append(
+            {
+                "id": item_id(case.id, name),
+                "kind": "judge",
+                "case_id": case.id,
+                "metric": name,
+                "scale": judge.scale,
+                "prompt": render_judge_prompt(
+                    judge,
+                    conversation=rendered.conversation,
+                    response=response,
+                    reference=case.reference,
+                    context=case.context,
+                    tool_calls=rendered.final_tool_calls or None,
+                    transcript=rendered.transcript,
+                    max_tool_result_chars=config.max_tool_result_chars,
+                ),
+            }
+        )
+    return items
+
+
+def apply_judge_results(grade: CaseGrade, results: dict[str, dict[str, Any]]) -> None:
+    """Fold runner results into the grade: error, mandatory fail, or quality miss."""
+    for name, entry in grade.judge_scores.items():
+        label = "custom metric" if entry.get("kind") == "custom" else "judge"
+        result = results.get(item_id(grade.id, name))
+        if result is None:
+            entry["error"] = "no result from judge runner"
+            grade.errors.append(f"error: {label} {name}: no result from judge runner")
+            continue
+        if result.get("error"):
+            entry["error"] = str(result["error"])
+            grade.errors.append(f"error: {label} {name}: {result['error']}")
+            continue
+        score = result.get("score")
+        if not isinstance(score, int | float) or isinstance(score, bool):
+            entry["error"] = f"{label} returned no numeric score ({score!r})"
+            grade.errors.append(f"error: {label} {name}: no numeric score")
+            continue
+        entry["score"] = float(score)
+        entry["reasoning"] = str(result.get("reasoning") or "")
+        passed = float(score) >= float(entry["threshold"])
+        entry["passed"] = passed
+        if passed:
+            continue
+        text = f"{name}: score {_fmt(score)} below threshold {_fmt(entry['threshold'])}"
+        if entry["quality"]:
+            grade.quality_misses.append(f"{text} (quality)")
+        else:
+            grade.failures.append(text)
+
+
+def _fmt(value: Any) -> str:
+    if isinstance(value, float) and value.is_integer():
+        return str(int(value))
+    return str(value)
+
+
+# --- aggregate ----------------------------------------------------------------
+
+
+def summarize(grades: list[CaseGrade]) -> dict[str, int]:
+    summary = dict.fromkeys(STATUSES, 0)
+    for grade in grades:
+        summary[grade.status] += 1
+    return summary
+
+
+def compute_quality(config: EvalConfig, grades: list[CaseGrade]) -> dict[str, dict[str, Any]]:
+    """Per quality metric: pass rate over the cases scored on it, and ``met``.
+
+    The denominator (``scored``) is the cases the metric actually scored: a
+    case that never declared the metric is not a pass. ``pass_rate`` and
+    ``met`` are null on an incomplete run (``status: incomplete``) and when no
+    case ran the metric (``status: not_run``, which cannot fail the gate: there
+    is nothing to measure, and every mandatory item still applies).
+    """
+    complete = bool(grades) and all(g.status not in (STATUS_ERROR, STATUS_MISSING) for g in grades)
+    quality: dict[str, dict[str, Any]] = {}
+    for name, metric in config.quality_metrics.items():
+        scored = [
+            entry
+            for g in grades
+            if (entry := g.judge_scores.get(name)) is not None
+            and entry.get("quality")
+            and entry.get("passed") is not None
+        ]
+        below = sum(1 for entry in scored if entry.get("passed") is False)
+        result: dict[str, Any] = {
+            "pass_rate": None,
+            "min_pass_rate": metric.min_pass_rate,
+            "met": None,
+            "below_threshold": below,
+            "scored": len(scored),
+            "passed": len(scored) - below,
+        }
+        if not complete:
+            result["status"] = "incomplete"
+        elif not scored:
+            result["status"] = "not_run"
+        else:
+            rate = (len(scored) - below) / len(scored)
+            result["pass_rate"] = round(rate, 4)
+            result["met"] = rate >= metric.min_pass_rate
+            result["status"] = "met" if result["met"] else "not_met"
+        quality[name] = result
+    return quality
+
+
+def exit_code_for(summary: dict[str, int], quality: dict[str, dict[str, Any]]) -> int:
+    if summary.get(STATUS_ERROR) or summary.get(STATUS_MISSING):
+        return EXIT_INCOMPLETE
+    if summary.get(STATUS_FAILED) or any(q.get("met") is False for q in quality.values()):
+        return EXIT_GATE_FAILED
+    return EXIT_OK
+
+
+def print_summary(console: Console, results: dict[str, Any]) -> None:
+    summary = results["summary"]
+    table = Table(title="Evaluation gate", show_header=True, header_style="bold")
+    table.add_column("Status")
+    table.add_column("Cases", justify="right")
+    for status in STATUSES:
+        count = summary.get(status, 0)
+        style = _STATUS_STYLE[status] if count else "dim"
+        table.add_row(f"[{style}]{status}[/{style}]", str(count))
+    table.add_row(
+        "[bold]planned[/bold]",
+        str(results.get("planned", sum(summary.get(s, 0) for s in STATUSES))),
+    )
+    console.print(table)
+
+    quality = results.get("quality") or {}
+    if quality:
+        qtable = Table(title="Quality metrics", show_header=True, header_style="bold")
+        qtable.add_column("Metric")
+        qtable.add_column("Pass rate", justify="right")
+        qtable.add_column("Passed/scored", justify="right")
+        qtable.add_column("Min", justify="right")
+        qtable.add_column("Met")
+        incomplete = bool(summary.get(STATUS_ERROR) or summary.get(STATUS_MISSING))
+        for name, entry in quality.items():
+            rate = entry.get("pass_rate")
+            met = entry.get("met")
+            if met is None:
+                not_run = entry.get("status") == "not_run" or (
+                    not incomplete and entry.get("scored") == 0
+                )
+                met_text = "n/a (no case ran it)" if not_run else "n/a (incomplete)"
+            else:
+                met_text = "[green]yes[/green]" if met else "[red]no[/red]"
+            scored = entry.get("scored")
+            qtable.add_row(
+                name,
+                "n/a" if rate is None else f"{rate:.0%}",
+                "n/a" if scored is None else f"{entry.get('passed', 0)}/{scored}",
+                f"{entry.get('min_pass_rate', 1.0):.0%}",
+                met_text,
+            )
+        console.print(qtable)
+
+    flagged = [c for c in results.get("cases", []) if c["status"] != STATUS_PASSED]
+    if flagged:
+        ctable = Table(title="Cases needing attention", show_header=True, header_style="bold")
+        ctable.add_column("Case")
+        ctable.add_column("Status")
+        ctable.add_column("Reasons")
+        for case in flagged:
+            style = _STATUS_STYLE[case["status"]]
+            ctable.add_row(
+                case["id"],
+                f"[{style}]{case['status']}[/{style}]",
+                "\n".join(case["reasons"][:4]) + ("\n..." if len(case["reasons"]) > 4 else ""),
+            )
+        console.print(ctable)
+    warnings = results.get("warnings") or []
+    for text in warnings:
+        console.print(f"[bold yellow]Warning:[/bold yellow] [yellow]{escape(text)}[/yellow]")
+    code = summary.get("exit_code", 0)
+    verdict = {
+        0: "[green]gate met[/green]",
+        1: "[red]gate failed[/red]",
+        2: "[magenta]incomplete run[/magenta]",
+    }
+    caveat = ""
+    if code == 0 and results.get("fake_model"):
+        caveat = (
+            " [bold yellow](fake model: plumbing check only, not a quality signal)[/bold yellow]"
+        )
+    console.print(f"Result: {verdict.get(code, code)} (exit code {code}){caveat}")

graph_agents_cli/eval/transcript.py

@@ -0,0 +1,192 @@
+# Copyright 2026 graph-agents-cli contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""What a judge sees of one case: the conversation, tool calls and replies of every turn.
+
+A multi-turn case sends each user message on one thread, and the trace keeps
+every turn under ``turns`` (the top-level ``response`` and ``tool_calls`` are
+the final turn's). A judge scoring the final reply needs the earlier replies
+and tool results: a follow-up that relies on an answer already given, or a
+claim grounded in an earlier tool result, would otherwise look wrong.
+
+Rendered pieces:
+
+* ``conversation``: every turn before the reply being scored in full (user
+  message, each tool call with its result, the agent's reply), then the final
+  user message. Dataset ``system``/``assistant`` messages, which are not sent
+  to the agent, appear in place and are labelled as such.
+* ``transcript``: the same plus the final turn's tool calls and reply.
+* the final turn's tool calls, rendered by ``render_tool_calls``.
+
+Tool results longer than the configured limit are cut with an explicit marker
+that tells the judge how much it did not see, so a claim drawn from the omitted
+part is never mistaken for an invented one.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+from graph_agents_cli.eval.dataset import EvalCase
+
+# Characters of one tool result shown to a judge before it is cut. The agent
+# read the whole result, so the judge needs it too; the bound only keeps one
+# runaway tool from exhausting the judge's context. Configurable per project as
+# ``judge.max_tool_result_chars`` in eval_config.yaml (null = never cut).
+DEFAULT_MAX_TOOL_RESULT_CHARS = 50_000
+
+NOT_SENT = "(from the dataset; not sent to the agent)"
+
+
+@dataclass
+class RenderedCase:
+    conversation: str
+    transcript: str
+    final_tool_calls: list[dict[str, Any]] = field(default_factory=list)
+    # Tool results (in any turn) cut for the judge, and how many characters went unseen.
+    truncated_results: int = 0
+    truncated_chars: int = 0
+    # Earlier turns whose replies the trace does not record (an older trace or a
+    # generate override): the judge is told, rather than shown a gap silently.
+    unrecorded_turns: int = 0
+
+    @property
+    def notes(self) -> list[str]:
+        notes: list[str] = []
+        if self.truncated_results:
+            notes.append(
+                f"{self.truncated_results} tool result(s) cut for the judge "
+                f"({self.truncated_chars} characters not shown; raise "
+                "judge.max_tool_result_chars in eval_config.yaml to show more)"
+            )
+        if self.unrecorded_turns:
+            notes.append(
+                f"{self.unrecorded_turns} earlier turn(s) have no recorded reply in the trace "
+                "(re-run eval generate to record every turn)"
+            )
+        return notes
+
+
+def _as_text(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    try:
+        return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
+    except (TypeError, ValueError):
+        return str(value)
+
+
+def _args_text(args: Any) -> str:
+    if args is None or args == {}:
+        return ""
+    return _as_text(args)
+
+
+def cut_tool_result(text: str, max_chars: int | None) -> tuple[str, int]:
+    """``(text shown to the judge, characters cut)``; the cut is marked in-band."""
+    if max_chars is None or len(text) <= max_chars:
+        return text, 0
+    omitted = len(text) - max_chars
+    marker = (
+        f" [TRUNCATED by graph-agents-cli: the judge sees the first {max_chars} of "
+        f"{len(text)} characters of this tool result; {omitted} more were returned to the "
+        "agent but are not shown here. Do not treat a claim as unsupported only because "
+        "it could come from the omitted part.]"
+    )
+    return text[:max_chars] + marker, omitted
+
+
+def render_tool_call(call: dict[str, Any], max_chars: int | None) -> tuple[str, int]:
+    """``name(args) -> result`` for one call, and how many result characters were cut."""
+    result, omitted = cut_tool_result(_as_text(call.get("result")), max_chars)
+    flag = " (error)" if call.get("is_error") else ""
+    return f"{call.get('name')}({_args_text(call.get('args'))}) -> {result}{flag}", omitted
+
+
+def render_tool_calls(
+    calls: list[dict[str, Any]] | None, max_chars: int | None
+) -> tuple[list[str], int, int]:
+    """Rendered lines, the number of results cut, and the characters cut."""
+    lines: list[str] = []
+    cut = 0
+    omitted_total = 0
+    for call in calls or []:
+        if not isinstance(call, dict):
+            continue
+        line, omitted = render_tool_call(call, max_chars)
+        lines.append(line)
+        if omitted:
+            cut += 1
+            omitted_total += omitted
+    return lines, cut, omitted_total
+
+
+def render_case(
+    case: EvalCase,
+    trace: dict[str, Any],
+    *,
+    max_tool_result_chars: int | None = DEFAULT_MAX_TOOL_RESULT_CHARS,
+) -> RenderedCase:
+    """Render the conversation and full transcript of one graded case."""
+    user_count = sum(1 for m in case.messages if m.get("role") == "user")
+    raw_turns = trace.get("turns")
+    turns: list[dict[str, Any]] = (
+        [t if isinstance(t, dict) else {} for t in raw_turns] if isinstance(raw_turns, list) else []
+    )
+    final_calls = [c for c in (trace.get("tool_calls") or []) if isinstance(c, dict)]
+    final_response = trace.get("response")
+    labelled = user_count > 1
+
+    before: list[str] = []  # everything up to and including the final user message
+    after: list[str] = []  # the final turn's tool calls and reply (transcript only)
+    rendered = RenderedCase(conversation="", transcript="", final_tool_calls=final_calls)
+
+    def _calls(lines_out: list[str], calls: list[dict[str, Any]]) -> None:
+        lines, cut, omitted = render_tool_calls(calls, max_tool_result_chars)
+        rendered.truncated_results += cut
+        rendered.truncated_chars += omitted
+        lines_out.extend(f"agent tool call: {line}" for line in lines)
+
+    turn_index = -1
+    for message in case.messages:
+        role = message.get("role", "?")
+        content = str(message.get("content", ""))
+        if role != "user":
+            # Dataset context for the judge; the agent never received it.
+            before.append(f"{role} {NOT_SENT}: {content}")
+            continue
+        turn_index += 1
+        if labelled:
+            before.append(f"--- turn {turn_index + 1} of {user_count} ---")
+        before.append(f"user: {content}")
+        if turn_index == user_count - 1:
+            continue  # the final turn's calls and reply are the ones being scored
+        record = turns[turn_index] if turn_index < len(turns) else None
+        if record is None:
+            rendered.unrecorded_turns += 1
+            before.append("agent: [reply not recorded in the trace]")
+            continue
+        _calls(before, [c for c in (record.get("tool_calls") or []) if isinstance(c, dict)])
+        before.append(f"agent: {_as_text(record.get('response'))}")
+
+    _calls(after, final_calls)
+    after.append(f"agent: {_as_text(final_response)}")
+
+    rendered.conversation = "\n".join(before)
+    rendered.transcript = "\n".join(before + after)
+    return rendered

graph_agents_cli/extension/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.