kiwimatecoder 0.1.0__py3-none-any.whl

kiwimatecoder/evals/cases.py

@@ -0,0 +1,230 @@
+"""Load and validate declarative eval cases from JSON files.
+
+A case is a small unit of expected agent behavior: a prompt, an optional
+starting workspace, and deterministic expectations about the run's text,
+tools, and resulting files. See ``evals/README.md`` for the schema.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from kiwimatecoder.config import has_path_anchor
+
+PACKAGE_ROOT = Path(__file__).resolve().parents[2]
+DEFAULT_CASES_DIR = PACKAGE_ROOT / "evals" / "cases"
+
+_TOP_LEVEL_KEYS = frozenset({"name", "description", "prompt", "files", "expect"})
+_EXPECT_KEYS = frozenset(
+    {
+        "text_contains",
+        "text_not_contains",
+        "tools_called",
+        "tools_not_called",
+        "files",
+        "success",
+    }
+)
+_LIST_FIELDS = (
+    "text_contains",
+    "text_not_contains",
+    "tools_called",
+    "tools_not_called",
+)
+
+
+class CaseError(ValueError):
+    """Raised when an eval case file is missing, unreadable, or malformed."""
+
+
+def _fail(source: Path | None, message: str) -> CaseError:
+    where = f"{source}: " if source is not None else ""
+    return CaseError(f"{where}{message}")
+
+
+def _non_empty_str(value: Any, *, key: str, source: Path | None) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise _fail(source, f"'{key}' must be a non-empty string")
+    return value
+
+
+def _string_list(value: Any, *, key: str, source: Path | None) -> tuple[str, ...]:
+    if not isinstance(value, list):
+        raise _fail(source, f"'{key}' must be a list of strings")
+    for item in value:
+        if not isinstance(item, str) or not item:
+            raise _fail(source, f"'{key}' entries must be non-empty strings")
+    return tuple(value)
+
+
+def _safe_relpath(value: Any, *, key: str, source: Path | None) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise _fail(source, f"'{key}' paths must be non-empty strings")
+    if "\\" in value:
+        raise _fail(source, f"'{key}' path {value!r} must use forward slashes")
+    path = Path(value)
+    if has_path_anchor(path) or value.startswith("~"):
+        raise _fail(source, f"'{key}' path {value!r} must be relative to the workspace")
+    if ".." in path.parts:
+        raise _fail(source, f"'{key}' path {value!r} must not contain '..'")
+    return value
+
+
+def _string_map(value: Any, *, key: str, source: Path | None) -> dict[str, str]:
+    if not isinstance(value, dict):
+        raise _fail(source, f"'{key}' must be an object mapping paths to contents")
+    result: dict[str, str] = {}
+    for raw_path, content in value.items():
+        relpath = _safe_relpath(raw_path, key=key, source=source)
+        if not isinstance(content, str):
+            raise _fail(source, f"'{key}' value for {relpath!r} must be a string")
+        result[relpath] = content
+    return result
+
+
+def _expect_files(value: Any, *, key: str, source: Path | None) -> dict[str, tuple[str, ...]]:
+    if not isinstance(value, dict):
+        raise _fail(
+            source, f"expect.'{key}' must be an object mapping paths to string lists"
+        )
+    result: dict[str, tuple[str, ...]] = {}
+    for raw_path, needles in value.items():
+        relpath = _safe_relpath(raw_path, key=f"expect.{key}", source=source)
+        result[relpath] = _string_list(needles, key=f"expect.{key}.{relpath}", source=source)
+    return result
+
+
+@dataclass
+class Expectation:
+    """Deterministic checks applied to one :class:`~kiwimatecoder.sdk.RunResult`."""
+
+    text_contains: tuple[str, ...] = ()
+    text_not_contains: tuple[str, ...] = ()
+    tools_called: tuple[str, ...] = ()
+    tools_not_called: tuple[str, ...] = ()
+    files: dict[str, tuple[str, ...]] = field(default_factory=dict)
+    success: bool | None = None
+
+
+@dataclass
+class EvalCase:
+    """One prompt plus the workspace files and expectations for its run."""
+
+    name: str
+    prompt: str
+    expect: Expectation = field(default_factory=Expectation)
+    description: str = ""
+    files: dict[str, str] = field(default_factory=dict)
+    source: Path | None = None
+
+
+def parse_case(payload: Any, *, source: Path | None = None) -> EvalCase:
+    """Validate an already-decoded case payload and return an :class:`EvalCase`."""
+    if not isinstance(payload, dict):
+        raise _fail(source, "case must be a JSON object")
+
+    unknown = sorted(set(payload) - _TOP_LEVEL_KEYS)
+    if unknown:
+        raise _fail(source, f"unknown key(s): {', '.join(unknown)}")
+
+    name = _non_empty_str(payload.get("name"), key="name", source=source)
+    prompt = _non_empty_str(payload.get("prompt"), key="prompt", source=source)
+
+    description = payload.get("description", "")
+    if not isinstance(description, str):
+        raise _fail(source, "'description' must be a string")
+
+    files: dict[str, str] = {}
+    if "files" in payload:
+        files = _string_map(payload["files"], key="files", source=source)
+
+    if "expect" not in payload:
+        raise _fail(source, "'expect' is required")
+    expect_payload = payload["expect"]
+    if not isinstance(expect_payload, dict):
+        raise _fail(source, "'expect' must be an object")
+    unknown_expect = sorted(set(expect_payload) - _EXPECT_KEYS)
+    if unknown_expect:
+        raise _fail(source, f"unknown expect key(s): {', '.join(unknown_expect)}")
+
+    success = expect_payload.get("success")
+    if success is not None and not isinstance(success, bool):
+        raise _fail(source, "'expect.success' must be a boolean")
+
+    expect = Expectation(
+        text_contains=_string_list(
+            expect_payload.get("text_contains", []), key="expect.text_contains", source=source
+        ),
+        text_not_contains=_string_list(
+            expect_payload.get("text_not_contains", []),
+            key="expect.text_not_contains",
+            source=source,
+        ),
+        tools_called=_string_list(
+            expect_payload.get("tools_called", []), key="expect.tools_called", source=source
+        ),
+        tools_not_called=_string_list(
+            expect_payload.get("tools_not_called", []),
+            key="expect.tools_not_called",
+            source=source,
+        ),
+        files=_expect_files(expect_payload.get("files", {}), key="files", source=source),
+        success=success,
+    )
+    return EvalCase(
+        name=name,
+        prompt=prompt,
+        expect=expect,
+        description=description,
+        files=files,
+        source=source,
+    )
+
+
+def load_case(path: str | Path) -> EvalCase:
+    """Read and validate one ``*.json`` case file."""
+    source = Path(path)
+    try:
+        raw = source.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise _fail(source, f"could not read case: {exc}") from exc
+    try:
+        payload = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise _fail(source, f"invalid JSON: {exc}") from exc
+    return parse_case(payload, source=source)
+
+
+def resolve_cases_dir(directory: str | Path | None = None) -> Path:
+    """Return the cases directory, falling back to ``./evals/cases`` when installed."""
+    if directory is not None:
+        return Path(directory).expanduser()
+    if DEFAULT_CASES_DIR.is_dir():
+        return DEFAULT_CASES_DIR
+    return Path.cwd() / "evals" / "cases"
+
+
+def discover_cases(directory: str | Path | None = None) -> list[EvalCase]:
+    """Load every ``*.json`` case under ``directory``, sorted by case name.
+
+    Raises :class:`CaseError` for a missing directory, a malformed case file,
+    or duplicate case names.
+    """
+    root = resolve_cases_dir(directory)
+    if not root.is_dir():
+        raise CaseError(f"Eval cases directory not found: {root}")
+    cases: list[EvalCase] = []
+    seen: dict[str, Path] = {}
+    for path in sorted(root.glob("*.json")):
+        case = load_case(path)
+        if case.name in seen:
+            raise CaseError(
+                f"Duplicate eval case name {case.name!r} in {path} and {seen[case.name]}"
+            )
+        seen[case.name] = path
+        cases.append(case)
+    cases.sort(key=lambda case: case.name.lower())
+    return cases

kiwimatecoder/evals/render.py

@@ -0,0 +1,46 @@
+"""Rich table and JSON rendering for eval reports."""
+
+from __future__ import annotations
+
+import json
+
+from rich.console import Console
+from rich.markup import escape
+from rich.table import Table
+
+from kiwimatecoder.evals.runner import EvalReport
+
+
+def report_to_json(report: EvalReport) -> str:
+    """Serialize a report as pretty-printed JSON."""
+    return json.dumps(report.to_dict(), indent=2)
+
+
+def build_report_table(report: EvalReport) -> Table:
+    """Build the per-case result table shown by ``kiwimatecoder eval run``."""
+    table = Table(title="Eval results")
+    table.add_column("Case", style="cyan", no_wrap=True)
+    table.add_column("Result", no_wrap=True)
+    table.add_column("Details", overflow="fold")
+    table.add_column("Time", justify="right", no_wrap=True)
+    for result in report.results:
+        label = "PASS" if result.success else "FAIL"
+        style = "green" if result.success else "red"
+        details = "; ".join(result.reasons)
+        table.add_row(
+            escape(result.name),
+            f"[{style}]{label}[/{style}]",
+            escape(details),
+            f"{result.duration_s:.1f}s",
+        )
+    return table
+
+
+def print_report(report: EvalReport, console: Console | None = None) -> None:
+    """Print the result table plus a one-line summary."""
+    console = console or Console()
+    console.print(build_report_table(report))
+    style = "green" if report.success else "red"
+    console.print(
+        f"[{style}]{report.passed}/{report.total} eval case(s) passed[/{style}]"
+    )

kiwimatecoder/evals/runner.py

@@ -0,0 +1,319 @@
+"""Run eval cases against the SDK and evaluate their expectations.
+
+``run_case`` materializes a case into a fresh temp workspace, drives
+:func:`kiwimatecoder.sdk.run_agent_sync`, and turns every expectation into a
+human-readable failure reason. ``run_suite`` runs a directory of cases and can
+write a JSON report. Neither raises for a failing case: only invalid input
+(missing directory, malformed case, no matching cases) raises ``CaseError``.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import threading
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from kiwimatecoder import config, sdk
+from kiwimatecoder.evals.cases import (
+    CaseError,
+    EvalCase,
+    discover_cases,
+    resolve_cases_dir,
+)
+from kiwimatecoder.tools.paths import atomic_write_text
+
+DEFAULT_TIMEOUT = 300.0
+DEFAULT_MAX_TURNS = 20
+
+
+class EvalTimeout(TimeoutError):
+    """Raised internally when a case exceeds its ``timeout``."""
+
+
+@dataclass
+class EvalResult:
+    """The outcome of one eval case: a pass/fail plus human-readable reasons."""
+
+    name: str
+    success: bool
+    reasons: list[str] = field(default_factory=list)
+    text: str = ""
+    tools_used: list[str] = field(default_factory=list)
+    error: str | None = None
+    duration_s: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "success": self.success,
+            "reasons": list(self.reasons),
+            "duration_s": round(self.duration_s, 3),
+            "error": self.error,
+            "tools_used": list(self.tools_used),
+            "text": self.text,
+        }
+
+
+@dataclass
+class EvalReport:
+    """The results of running a suite of eval cases."""
+
+    results: list[EvalResult] = field(default_factory=list)
+    provider: str | None = None
+    model: str | None = None
+    cases_dir: str = ""
+
+    @property
+    def total(self) -> int:
+        return len(self.results)
+
+    @property
+    def passed(self) -> int:
+        return sum(1 for result in self.results if result.success)
+
+    @property
+    def failed(self) -> int:
+        return self.total - self.passed
+
+    @property
+    def success(self) -> bool:
+        return self.failed == 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "success": self.success,
+            "total": self.total,
+            "passed": self.passed,
+            "failed": self.failed,
+            "provider": self.provider,
+            "model": self.model,
+            "cases_dir": self.cases_dir,
+            "results": [result.to_dict() for result in self.results],
+        }
+
+
+def materialize_case(case: EvalCase, workspace: str | Path) -> dict[str, Path]:
+    """Write the case's starting files into ``workspace`` and return their paths."""
+    root = Path(workspace)
+    written: dict[str, Path] = {}
+    for relpath in sorted(case.files):
+        target = root / relpath
+        target.parent.mkdir(parents=True, exist_ok=True)
+        atomic_write_text(target, case.files[relpath])
+        written[relpath] = target
+    return written
+
+
+def _describe_tools(tools: list[str]) -> str:
+    return ", ".join(tools) if tools else "none"
+
+
+def evaluate_case(case: EvalCase, result: sdk.RunResult, workspace: str | Path) -> list[str]:
+    """Return human-readable failures for a completed run (empty means passed)."""
+    expect = case.expect
+    problems: list[str] = []
+
+    for needle in expect.text_contains:
+        if needle not in result.text:
+            problems.append(f"text_contains: {needle!r} not found in the reply")
+    for needle in expect.text_not_contains:
+        if needle in result.text:
+            problems.append(f"text_not_contains: {needle!r} found in the reply")
+
+    used = list(result.tools_used)
+    for tool in expect.tools_called:
+        if tool not in used:
+            problems.append(
+                f"tools_called: {tool!r} was not used (used: {_describe_tools(used)})"
+            )
+    for tool in expect.tools_not_called:
+        if tool in used:
+            problems.append(f"tools_not_called: {tool!r} was used")
+
+    root = Path(workspace)
+    for relpath in sorted(expect.files):
+        target = root / relpath
+        if not target.is_file():
+            problems.append(f"files: {relpath!r} was not created")
+            continue
+        content = target.read_text(encoding="utf-8", errors="replace")
+        for needle in expect.files[relpath]:
+            if needle not in content:
+                problems.append(f"files: {relpath!r} does not contain {needle!r}")
+
+    if expect.success is not None:
+        if result.success != expect.success:
+            problems.append(f"success: expected {expect.success}, got {result.success}")
+    elif result.error:
+        problems.append(f"run failed: {result.error}")
+    return problems
+
+
+def _run_with_timeout(
+    prompt: str,
+    *,
+    workspace: Path,
+    provider: str | None,
+    model: str | None,
+    mode: str,
+    timeout: float,
+    max_turns: int,
+) -> sdk.RunResult:
+    """Run the SDK on a worker thread so ``timeout`` can bound the wait.
+
+    The worker is a daemon thread, so a timed-out run never blocks process
+    exit; its result is discarded.
+    """
+    box: list[sdk.RunResult] = []
+    errors: list[BaseException] = []
+
+    def target() -> None:
+        try:
+            box.append(
+                sdk.run_agent_sync(
+                    prompt,
+                    workspace=workspace,
+                    provider=provider,
+                    model=model,
+                    mode=mode,
+                    max_turns=max_turns,
+                )
+            )
+        except BaseException as exc:  # re-raised on the caller's thread
+            errors.append(exc)
+
+    worker = threading.Thread(target=target, name="kiwimatecoder-eval", daemon=True)
+    worker.start()
+    worker.join(timeout=timeout)
+    if worker.is_alive():
+        raise EvalTimeout(f"timed out after {timeout:g}s")
+    if errors:
+        raise errors[0]
+    if not box:
+        raise RuntimeError("eval run produced no result")
+    return box[0]
+
+
+def run_case(
+    case: EvalCase,
+    *,
+    provider: str | None = None,
+    model: str | None = None,
+    mode: str = "auto-accept",
+    timeout: float = DEFAULT_TIMEOUT,
+    max_turns: int = DEFAULT_MAX_TURNS,
+) -> EvalResult:
+    """Materialize ``case``, run the SDK, and evaluate every expectation.
+
+    A run error, provider exception, or timeout becomes a failed result with a
+    reason; this function does not raise for a failing case.
+    """
+    started = time.monotonic()
+    with tempfile.TemporaryDirectory(prefix="kiwimatecoder-eval-") as tmp:
+        workspace = Path(tmp)
+        try:
+            materialize_case(case, workspace)
+        except OSError as exc:
+            return EvalResult(
+                name=case.name,
+                success=False,
+                reasons=[f"could not materialize case files: {exc}"],
+                duration_s=time.monotonic() - started,
+            )
+        try:
+            result = _run_with_timeout(
+                case.prompt,
+                workspace=workspace,
+                provider=provider,
+                model=model,
+                mode=mode,
+                timeout=timeout,
+                max_turns=max_turns,
+            )
+        except EvalTimeout as exc:
+            return EvalResult(
+                name=case.name,
+                success=False,
+                reasons=[str(exc)],
+                error="timeout",
+                duration_s=time.monotonic() - started,
+            )
+        except Exception as exc:
+            message = f"{type(exc).__name__}: {exc}"
+            return EvalResult(
+                name=case.name,
+                success=False,
+                reasons=[f"run failed: {message}"],
+                error=message,
+                duration_s=time.monotonic() - started,
+            )
+        reasons = evaluate_case(case, result, workspace)
+        return EvalResult(
+            name=case.name,
+            success=not reasons,
+            reasons=reasons,
+            text=result.text,
+            tools_used=list(result.tools_used),
+            error=result.error,
+            duration_s=time.monotonic() - started,
+        )
+
+
+def run_suite(
+    cases_dir: str | Path | None = None,
+    *,
+    provider: str | None = None,
+    model: str | None = None,
+    mode: str = "auto-accept",
+    filter: str | None = None,
+    report_path: str | Path | None = None,
+    timeout: float = DEFAULT_TIMEOUT,
+    max_turns: int = DEFAULT_MAX_TURNS,
+) -> EvalReport:
+    """Run every matching case and optionally write a JSON report.
+
+    Raises :class:`CaseError` for invalid input: a missing directory, no
+    matching cases, malformed case files, or an unknown provider / no model
+    passed or chosen for it. A case whose expectations fail is reported, never
+    raised.
+    """
+    root = resolve_cases_dir(cases_dir)
+    cases = discover_cases(root)
+    if filter:
+        needle = filter.lower()
+        cases = [case for case in cases if needle in case.name.lower()]
+    if not cases:
+        if filter:
+            raise CaseError(f"No eval cases matched {filter!r} in {root}")
+        raise CaseError(f"No eval cases found in {root}")
+    # One clear error up front instead of every case failing the same way.
+    try:
+        config.require_model(
+            config.get_provider_config(provider or config.get_selected_provider_id()),
+            override=model,
+        )
+    except (KeyError, ValueError) as exc:
+        raise CaseError(str(exc)) from exc
+
+    report = EvalReport(provider=provider, model=model, cases_dir=str(root))
+    for case in cases:
+        report.results.append(
+            run_case(
+                case,
+                provider=provider,
+                model=model,
+                mode=mode,
+                timeout=timeout,
+                max_turns=max_turns,
+            )
+        )
+
+    if report_path is not None:
+        target = Path(report_path)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        atomic_write_text(target, json.dumps(report.to_dict(), indent=2) + "\n")
+    return report

kiwimatecoder/events.py

@@ -0,0 +1,95 @@
+"""A tiny synchronous event bus for lifecycle notifications.
+
+Dependency-free by design: subscribers receive an :class:`Event` and may return
+any value. :meth:`EventBus.emit` calls them in registration order and collects
+their results. A raising subscriber never breaks the caller -- the exception is
+captured in the results list and dispatch continues to the next subscriber.
+
+Module-level helpers proxy to a default :data:`BUS`, which the agent and REPL
+use unless a fresh bus is injected (tests do this for isolation).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+SESSION_START = "session_start"
+SESSION_END = "session_end"
+PRE_TOOL = "pre_tool"
+POST_TOOL = "post_tool"
+
+EVENT_NAMES = (SESSION_START, SESSION_END, PRE_TOOL, POST_TOOL)
+
+Subscriber = Callable[["Event"], Any]
+Unsubscribe = Callable[[], None]
+
+
+@dataclass
+class Event:
+    """One named lifecycle notification with a free-form payload."""
+
+    name: str
+    payload: dict[str, Any] = field(default_factory=dict)
+
+
+class EventBus:
+    """Synchronous publish/subscribe over named events."""
+
+    def __init__(self) -> None:
+        self._subscribers: dict[str, list[Subscriber]] = {}
+
+    def subscribe(self, name: str, callback: Subscriber) -> Unsubscribe:
+        """Register ``callback`` for ``name``; returns an unsubscribe callable."""
+        self._subscribers.setdefault(name, []).append(callback)
+
+        def unsubscribe() -> None:
+            self.unsubscribe(name, callback)
+
+        return unsubscribe
+
+    def unsubscribe(self, name: str, callback: Subscriber) -> bool:
+        """Remove one subscription; returns whether it was registered."""
+        callbacks = self._subscribers.get(name)
+        if not callbacks:
+            return False
+        for index, registered in enumerate(callbacks):
+            if registered is callback:
+                del callbacks[index]
+                if not callbacks:
+                    self._subscribers.pop(name, None)
+                return True
+        return False
+
+    def emit(self, name: str, **payload: Any) -> list[Any]:
+        """Call every subscriber for ``name``; collect results and exceptions."""
+        event = Event(name=name, payload=payload)
+        results: list[Any] = []
+        for callback in list(self._subscribers.get(name, [])):
+            try:
+                results.append(callback(event))
+            except Exception as exc:  # a bad subscriber must not break emit
+                results.append(exc)
+        return results
+
+    def clear(self) -> None:
+        """Drop every subscription."""
+        self._subscribers.clear()
+
+
+BUS = EventBus()
+
+
+def subscribe(name: str, callback: Subscriber) -> Unsubscribe:
+    """Subscribe to ``name`` on the default bus."""
+    return BUS.subscribe(name, callback)
+
+
+def unsubscribe(name: str, callback: Subscriber) -> bool:
+    """Unsubscribe from ``name`` on the default bus."""
+    return BUS.unsubscribe(name, callback)
+
+
+def emit(name: str, **payload: Any) -> list[Any]:
+    """Emit ``name`` on the default bus."""
+    return BUS.emit(name, **payload)