python-yama 0.1.0__py3-none-any.whl

yama/cli.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+from io import StringIO
+from pathlib import Path
+from typing import Any
+
+from rich import box
+from rich.console import Console, Group
+from rich.table import Table
+from rich.text import Text
+from rich.tree import Tree
+
+from .workspace import (
+    ConfigurationError,
+    collect_case_paths,
+    load_workspace_config,
+    resolve_plugin_root,
+)
+from .llm import LLMError, LiteLLMClient, ScriptedLLMClient
+from .loaders import CollectionError, resolve_case
+from .report import write_html_report
+from .runner import YamaRunner
+
+DEFAULT_REPORT = Path("__yama_default_report__")
+
+
+def _status(passed: bool, *, compact: bool = False) -> Text:
+    if passed:
+        return Text("✓" if compact else "PASS", style="bold green")
+    return Text("✗" if compact else "FAIL", style="bold red")
+
+
+def _summary_case_key(case_key: str) -> str:
+    prefix = "__evals__/cases/"
+    return case_key.removeprefix(prefix)
+
+
+def build_results_renderable(results: list[Any]) -> Group:
+    table = Table(
+        title="Yama Eval Results",
+        box=box.SIMPLE_HEAVY,
+        header_style="bold cyan",
+        title_style="bold",
+        expand=True,
+    )
+    table.add_column("Result", no_wrap=True)
+    table.add_column("Case", ratio=1, overflow="fold")
+    table.add_column("Runs", justify="right", no_wrap=True)
+    table.add_column("Pass rate", justify="right", no_wrap=True)
+
+    passed_runs = 0
+    total_runs = 0
+    for result in results:
+        case_passed_runs = sum(run.passed for run in result.runs)
+        passed_runs += case_passed_runs
+        total_runs += len(result.runs)
+        table.add_row(
+            _status(result.passed),
+            Text(_summary_case_key(result.case_key)),
+            f"{case_passed_runs}/{len(result.runs)}",
+            f"{result.pass_rate:.0%}",
+        )
+
+    passed_cases = sum(result.passed for result in results)
+    table.caption = (
+        f"{passed_cases}/{len(results)} cases passed  •  "
+        f"{passed_runs}/{total_runs} runs passed"
+    )
+    table.caption_style = "green" if passed_cases == len(results) else "red"
+
+    details = Tree(Text("Details", style="bold"), guide_style="dim")
+    for result in results:
+        case_label = Text.assemble(
+            _status(result.passed, compact=True),
+            " ",
+            (result.case_key, "bold"),
+            (f"  {result.pass_rate:.0%}", "dim"),
+        )
+        case_node = details.add(case_label)
+        for run in result.runs:
+            run_label = Text.assemble(
+                _status(run.passed, compact=True),
+                " ",
+                (f"Run {run.run_index}", "bold"),
+            )
+            run_node = case_node.add(run_label)
+            if run.error:
+                run_node.add(Text.assemble(("Error: ", "bold red"), (run.error, "red")))
+            for step in run.steps:
+                failed = [
+                    check
+                    for check in step.hard_checks
+                    if not check.get("passed", False)
+                ]
+                judge = step.judge_result
+                judge_passed = judge is None or bool(judge.get("passed", False))
+                step_passed = not failed and step.hard_pass and judge_passed
+                hard_passed = len(step.hard_checks) - len(failed)
+                step_label = Text.assemble(
+                    _status(step_passed, compact=True),
+                    " ",
+                    (step.step_id, "bold"),
+                    (
+                        f"  hard checks {hard_passed}/{len(step.hard_checks)}",
+                        "dim",
+                    ),
+                )
+                if judge is not None:
+                    step_label.append("  judge ", style="dim")
+                    step_label.append(
+                        "passed" if judge_passed else "failed",
+                        style="green" if judge_passed else "bold red",
+                    )
+                    if isinstance(judge.get("overall"), (int, float)):
+                        step_label.append(f" {judge['overall']:.0%}", style="dim")
+                step_node = run_node.add(step_label)
+                if judge is not None and not judge_passed:
+                    judge_node = step_node.add(Text("Judge dimensions", style="bold red"))
+                    for dimension in judge.get("dimensions", []):
+                        dimension_label = Text.assemble(
+                            (str(dimension.get("name", "unnamed")), "bold"),
+                            f": {dimension.get('score', '?')}/5",
+                        )
+                        if dimension.get("reason"):
+                            dimension_label.append(
+                                f"  {dimension['reason']}", style="dim"
+                            )
+                        judge_node.add(dimension_label)
+                for index, check in enumerate(step.hard_checks, 1):
+                    label = check.get("label") or check.get("type", "hard check")
+                    if check.get("passed", False):
+                        step_node.add(
+                            Text.assemble(
+                                _status(True, compact=True),
+                                " ",
+                                (f"{index:02d} {label}", "green"),
+                            )
+                        )
+                        continue
+                    expected = check.get("expected", "assertion passes")
+                    actual = check.get("actual", check.get("detail", "unknown"))
+                    reason = check.get("reason", check.get("detail", "assertion failed"))
+                    check_node = step_node.add(
+                        Text.assemble(
+                            _status(False, compact=True),
+                            " ",
+                            (f"{index:02d} {label}", "bold red"),
+                        )
+                    )
+                    check_node.add(
+                        Text.assemble(("Expected  ", "bold cyan"), str(expected))
+                    )
+                    check_node.add(
+                        Text.assemble(("Actual    ", "bold yellow"), str(actual))
+                    )
+                    check_node.add(
+                        Text.assemble(("Reason    ", "bold red"), str(reason))
+                    )
+
+    return Group(table, Text(), details)
+
+
+def format_text_results(results: list[Any]) -> str:
+    """Render deterministic, color-free output for tests and non-console callers."""
+    output = StringIO()
+    console = Console(
+        file=output,
+        color_system=None,
+        force_terminal=False,
+        width=120,
+    )
+    console.print(build_results_renderable(results))
+    return output.getvalue().rstrip()
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="yama", description="Run Agent Skill eval cases"
+    )
+    parser.add_argument("paths", nargs="*", type=Path, help="Explicit YAML case paths")
+    parser.add_argument(
+        "--plugin-root", type=Path, help="Run a single plugin by root path"
+    )
+    parser.add_argument(
+        "--plugin", action="append", help="Plugin name from yama.toml"
+    )
+    parser.add_argument(
+        "--all-plugins", action="store_true", help="Run all configured plugins"
+    )
+    parser.add_argument(
+        "--response-script", type=Path, help="Use deterministic scripted LLM responses"
+    )
+    parser.add_argument("--result-dir", type=Path, help="Override artifact directory")
+    parser.add_argument(
+        "--report",
+        type=Path,
+        nargs="?",
+        const=DEFAULT_REPORT,
+        metavar="PATH",
+        help=(
+            "Write a standalone HTML execution report"
+            " (default: reports/latest.html beside the artifact runs directory)"
+        ),
+    )
+    parser.add_argument(
+        "--list", action="store_true", help="Collect and list cases without running"
+    )
+    parser.add_argument(
+        "--no-artifacts", action="store_true", help="Do not write run artifacts"
+    )
+    parser.add_argument("--json", action="store_true", help="Print JSON results")
+    return parser
+
+
+async def _run(args: argparse.Namespace) -> int:
+    workspace = load_workspace_config(Path.cwd())
+    targets: list[tuple[str | None, Path, dict[str, Any]]] = []
+    if args.plugin_root:
+        root, plugin_config = resolve_plugin_root(
+            workspace, explicit_root=args.plugin_root
+        )
+        targets.append((None, root, plugin_config))
+    elif args.all_plugins:
+        if not workspace.plugins:
+            raise ConfigurationError("yama.toml has no configured plugins")
+        for name in workspace.plugins:
+            root, plugin_config = resolve_plugin_root(workspace, plugin_name=name)
+            targets.append((name, root, plugin_config))
+    elif args.plugin:
+        for name in args.plugin:
+            root, plugin_config = resolve_plugin_root(workspace, plugin_name=name)
+            targets.append((name, root, plugin_config))
+    else:
+        root, plugin_config = resolve_plugin_root(workspace)
+        targets.append((None, root, plugin_config))
+
+    collected = []
+    for _, plugin_root, plugin_config in targets:
+        paths = collect_case_paths(
+            plugin_root,
+            workspace,
+            plugin_config,
+            explicit_paths=args.paths or None,
+        )
+        for path in paths:
+            collected.append(resolve_case(path, plugin_root, workspace, plugin_config))
+    if args.list:
+        for case in collected:
+            print(case.case_key)
+        return 0
+    if not collected:
+        Console(stderr=True).print("[bold red]No eval cases collected.[/bold red]")
+        return 2
+
+    llm = (
+        ScriptedLLMClient.from_file(args.response_script.resolve())
+        if args.response_script
+        else LiteLLMClient()
+    )
+    all_results = []
+    result_roots = []
+    for case in collected:
+        result_root = (
+            args.result_dir.resolve()
+            if args.result_dir
+            else case.plugin_root / workspace.result_dir
+        )
+        result_roots.append(result_root)
+        runner = YamaRunner(
+            llm,
+            result_root=result_root,
+            write_artifacts=not args.no_artifacts,
+        )
+        all_results.append(await runner.run_case(case))
+    if isinstance(llm, ScriptedLLMClient):
+        llm.assert_consumed()
+    report_target = args.report
+    if report_target == DEFAULT_REPORT:
+        report_target = result_roots[0].parent / "reports" / "latest.html"
+    report_path = (
+        write_html_report(collected, all_results, report_target)
+        if report_target
+        else None
+    )
+    if args.json:
+        print(
+            json.dumps(
+                [result.to_dict() for result in all_results],
+                ensure_ascii=False,
+                indent=2,
+            )
+        )
+    else:
+        Console().print(build_results_renderable(all_results))
+    if report_path:
+        Console(stderr=args.json).print(
+            Text.assemble(
+                ("HTML report: ", "bold"),
+                (str(report_path), f"cyan link {report_path.as_uri()}"),
+            )
+        )
+    return 0 if all(result.passed for result in all_results) else 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    try:
+        return asyncio.run(_run(args))
+    except (ConfigurationError, CollectionError, LLMError, ValueError) as exc:
+        Console(stderr=True).print(
+            Text.assemble(("Collection error: ", "bold red"), (str(exc), "red"))
+        )
+        return 2
+    except KeyboardInterrupt:
+        return 130

yama/llm.py

@@ -0,0 +1,365 @@
+from __future__ import annotations
+
+import base64
+import json
+import os
+from abc import ABC, abstractmethod
+from copy import deepcopy
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+import yaml
+
+from .models import LLMResponse, ToolCall, ToolSchema
+
+
+class LLMError(RuntimeError):
+    pass
+
+
+def _looks_like_openai_model(model: str) -> bool:
+    return model.startswith(("gpt-", "chatgpt-", "o1", "o3", "o4"))
+
+
+class LLMClient(ABC):
+    @abstractmethod
+    async def complete(
+        self,
+        *,
+        messages: list[dict[str, Any]],
+        tools: list[ToolSchema],
+        model: dict[str, Any],
+    ) -> LLMResponse:
+        raise NotImplementedError
+
+
+def _data_url(path: Path, media_type: str) -> str:
+    encoded = base64.b64encode(path.read_bytes()).decode("ascii")
+    return f"data:{media_type};base64,{encoded}"
+
+
+def _file_delivery(part: dict[str, Any]) -> str:
+    delivery = part.get("delivery", "file_path_inline")
+    return {
+        "metadata": "file_path_inline",
+        "content": "file_content",
+    }.get(delivery, delivery)
+
+
+def _litellm_content(content: Any) -> str | list[dict[str, Any]]:
+    if isinstance(content, str):
+        return content
+    inline_file_paths = [
+        part["fixture"]
+        for part in content
+        if part["type"] in {"input_image", "input_file"}
+        and _file_delivery(part) == "file_path_inline"
+    ]
+    inline_files_text = "\n".join(
+        ["Files:", *(f"- {path}" for path in inline_file_paths)]
+    )
+    has_text_part = any(part["type"] == "text" for part in content)
+    inline_files_added = False
+    parts: list[dict[str, Any]] = []
+    for part in content:
+        if part["type"] == "text":
+            text = part["text"]
+            if inline_file_paths and not inline_files_added:
+                text = (
+                    f"{text.rstrip()}\n{inline_files_text}"
+                    if text
+                    else inline_files_text
+                )
+                inline_files_added = True
+            parts.append({"type": "text", "text": text})
+        elif part["type"] == "input_image" and _file_delivery(part) == "file_content":
+            parts.append(
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": _data_url(
+                            Path(part["fixture"]), part["media_type"]
+                        )
+                    },
+                }
+            )
+        elif (
+            part["type"] in {"input_image", "input_file"}
+            and _file_delivery(part) == "file_path_inline"
+        ):
+            if not has_text_part and not inline_files_added:
+                parts.append({"type": "text", "text": inline_files_text})
+                inline_files_added = True
+        elif part["type"] == "input_file" and _file_delivery(part) == "file_content":
+            encoded = base64.b64encode(Path(part["fixture"]).read_bytes()).decode(
+                "ascii"
+            )
+            parts.append(
+                {
+                    "type": "file",
+                    "file": {
+                        "file_data": (
+                            f'data:{part["media_type"]};base64,{encoded}'
+                        ),
+                    },
+                }
+            )
+    if len(parts) == 1 and parts[0]["type"] == "text":
+        return parts[0]["text"]
+    return parts
+
+
+def _litellm_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    converted: list[dict[str, Any]] = []
+    for message in messages:
+        converted_message = deepcopy(message)
+        converted_message["content"] = _litellm_content(
+            converted_message.get("content", "")
+        )
+        if converted_message["role"] == "assistant" and converted_message.get(
+            "tool_calls"
+        ):
+            for call in converted_message["tool_calls"]:
+                arguments = call["function"].get("arguments", {})
+                if not isinstance(arguments, str):
+                    call["function"]["arguments"] = json.dumps(
+                        arguments, ensure_ascii=False
+                    )
+        converted.append(converted_message)
+    return converted
+
+
+def _resolve_litellm_model(
+    model: Mapping[str, Any], environment: Mapping[str, str]
+) -> tuple[str, str]:
+    raw_name = model.get("name")
+    if not isinstance(raw_name, str) or not raw_name:
+        raise LLMError("model.name is required")
+    explicit_provider = model.get("provider") or environment.get(
+        "YAMA_MODEL_PROVIDER"
+    )
+    if explicit_provider is not None and not isinstance(explicit_provider, str):
+        raise LLMError("model.provider must be a string")
+
+    if explicit_provider:
+        prefix = f"{explicit_provider}/"
+        resolved = raw_name if raw_name.startswith(prefix) else f"{prefix}{raw_name}"
+        return resolved, explicit_provider
+    if "/" in raw_name:
+        return raw_name, raw_name.split("/", 1)[0]
+    if (
+        environment.get("OPENROUTER_API_KEY") or environment.get("OR_API_KEY")
+    ) and not environment.get("OPENAI_API_KEY"):
+        routed = f"openai/{raw_name}" if _looks_like_openai_model(raw_name) else raw_name
+        return f"openrouter/{routed}", "openrouter"
+    return f"openai/{raw_name}", "openai"
+
+
+def _litellm_call_arguments(
+    *,
+    messages: list[dict[str, Any]],
+    tools: list[ToolSchema],
+    model: Mapping[str, Any],
+    environment: Mapping[str, str],
+) -> tuple[dict[str, Any], dict[str, Any]]:
+    resolved_model, provider = _resolve_litellm_model(model, environment)
+    options = deepcopy(dict(model))
+    options.pop("name", None)
+    options.pop("provider", None)
+    api_base = options.pop("api_base", None) or options.pop("base_url", None)
+    arguments: dict[str, Any] = {
+        "model": resolved_model,
+        "messages": _litellm_messages(messages),
+    }
+    if tools:
+        arguments["tools"] = [tool.for_provider() for tool in tools]
+    if api_base:
+        arguments["api_base"] = api_base
+    arguments.update({key: value for key, value in options.items() if value is not None})
+    return arguments, {"name": provider, "model": resolved_model}
+
+
+def _value(item: Any, key: str, default: Any = None) -> Any:
+    if isinstance(item, dict):
+        return item.get(key, default)
+    return getattr(item, key, default)
+
+
+class LiteLLMClient(LLMClient):
+    def __init__(
+        self,
+        completion_callable: Callable[..., Any] | None = None,
+        *,
+        environment: Mapping[str, str] | None = None,
+    ) -> None:
+        self.completion_callable = completion_callable
+        self.environment = environment if environment is not None else os.environ
+        self.last_provider: dict[str, Any] | None = None
+        self.last_request: dict[str, Any] | None = None
+
+    async def complete(
+        self,
+        *,
+        messages: list[dict[str, Any]],
+        tools: list[ToolSchema],
+        model: dict[str, Any],
+    ) -> LLMResponse:
+        self.last_request = None
+        try:
+            arguments, provider = _litellm_call_arguments(
+                messages=messages,
+                tools=tools,
+                model=model,
+                environment=self.environment,
+            )
+            self.last_provider = provider
+            self.last_request = {
+                "method": "litellm.acompletion",
+                "arguments": deepcopy(arguments),
+            }
+            if self.completion_callable is None:
+                from litellm import acompletion
+
+                response = await acompletion(**arguments)
+            else:
+                response = await self.completion_callable(**arguments)
+        except Exception as exc:
+            target = (
+                self.last_provider["model"]
+                if self.last_provider is not None
+                else "unresolved model"
+            )
+            raise LLMError(f"LiteLLM request failed for {target!r}: {exc}") from exc
+
+        choices = _value(response, "choices", [])
+        if not choices:
+            raise LLMError("LiteLLM response has no choices")
+        message = _value(choices[0], "message")
+        if message is None:
+            raise LLMError("LiteLLM response choice has no message")
+        calls: list[ToolCall] = []
+        for call in _value(message, "tool_calls", []) or []:
+            function = _value(call, "function", {})
+            raw_arguments = _value(function, "arguments", "{}")
+            try:
+                arguments = (
+                    raw_arguments
+                    if isinstance(raw_arguments, dict)
+                    else json.loads(raw_arguments or "{}")
+                )
+            except json.JSONDecodeError as exc:
+                raise LLMError(
+                    "model returned invalid JSON arguments for "
+                    f"{_value(function, 'name')}: {raw_arguments}"
+                ) from exc
+            calls.append(
+                ToolCall(
+                    str(_value(call, "id", "")),
+                    str(_value(function, "name", "")),
+                    arguments,
+                )
+            )
+        content = _value(message, "content", "") or ""
+        if not isinstance(content, str):
+            content = "\n".join(
+                str(_value(part, "text", ""))
+                for part in content
+                if _value(part, "text") is not None
+            )
+        raw = (
+            response.model_dump() if hasattr(response, "model_dump") else str(response)
+        )
+        return LLMResponse(content=content, tool_calls=calls, raw=raw)
+
+
+class ScriptedLLMClient(LLMClient):
+    """Deterministic LLM adapter for framework tests and replay fixtures."""
+
+    def __init__(self, responses: list[dict[str, Any]]) -> None:
+        self.responses = deepcopy(responses)
+        self.index = 0
+        self.requests: list[dict[str, Any]] = []
+        self.last_request: dict[str, Any] | None = None
+        self.last_provider: dict[str, Any] | None = None
+
+    @classmethod
+    def from_file(cls, path: Path) -> "ScriptedLLMClient":
+        raw = yaml.safe_load(path.read_text(encoding="utf-8"))
+        responses = raw.get("responses") if isinstance(raw, dict) else raw
+        if not isinstance(responses, list):
+            raise LLMError(f"script must contain a response list: {path}")
+        return cls(responses)
+
+    async def complete(
+        self,
+        *,
+        messages: list[dict[str, Any]],
+        tools: list[ToolSchema],
+        model: dict[str, Any],
+    ) -> LLMResponse:
+        if self.index >= len(self.responses):
+            raise LLMError("scripted LLM response list is exhausted")
+        request, provider = _litellm_call_arguments(
+            messages=messages,
+            tools=tools,
+            model=model,
+            environment=os.environ,
+        )
+        self.last_provider = provider
+        self.last_request = {
+            "method": "litellm.acompletion",
+            "arguments": deepcopy(request),
+        }
+        self.requests.append(request)
+        scripted = self.responses[self.index]
+        self.index += 1
+        if not isinstance(scripted, dict):
+            raise LLMError(f"scripted response {self.index} must be an object")
+        self._check_expectation(scripted.get("expect"), request)
+        calls: list[ToolCall] = []
+        for index, call in enumerate(scripted.get("tool_calls", []), 1):
+            calls.append(
+                ToolCall(
+                    id=str(call.get("id", f"script-call-{self.index}-{index}")),
+                    name=call["name"],
+                    arguments=deepcopy(call.get("arguments", {})),
+                )
+            )
+        return LLMResponse(
+            content=str(scripted.get("content", "")),
+            tool_calls=calls,
+            raw=deepcopy(scripted),
+        )
+
+    def _check_expectation(self, expectation: Any, request: dict[str, Any]) -> None:
+        if expectation is None:
+            return
+        if not isinstance(expectation, dict):
+            raise LLMError("script expect must be an object")
+        if "tools" in expectation:
+            actual = [tool["function"]["name"] for tool in request["tools"]]
+            if actual != expectation["tools"]:
+                raise LLMError(
+                    f"script expected tools {expectation['tools']}, got {actual}"
+                )
+        if "last_role" in expectation:
+            actual_role = request["messages"][-1]["role"]
+            if actual_role != expectation["last_role"]:
+                raise LLMError(
+                    f"script expected last role {expectation['last_role']!r}, got {actual_role!r}"
+                )
+        if "last_user_contains" in expectation:
+            user_messages = [
+                item for item in request["messages"] if item["role"] == "user"
+            ]
+            text = json.dumps(user_messages[-1]["content"], ensure_ascii=False)
+            if expectation["last_user_contains"] not in text:
+                raise LLMError(
+                    f"last user message does not contain {expectation['last_user_contains']!r}"
+                )
+
+    def assert_consumed(self) -> None:
+        if self.index != len(self.responses):
+            raise LLMError(
+                f"scripted LLM has {len(self.responses) - self.index} unconsumed responses"
+            )

yama/loaders/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from .case import resolve_case
+from .common import CollectionError, load_yaml
+
+__all__ = ["CollectionError", "load_yaml", "resolve_case"]