python-yama 0.1.0__py3-none-any.whl

yama/loaders/case.py

@@ -0,0 +1,378 @@
+from __future__ import annotations
+
+import mimetypes
+from copy import deepcopy
+from html import escape
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from ..workspace import WorkspaceConfig
+from ..models import LoadedSkill, ResolvedCase
+from .common import CollectionError, load_yaml, sha256_file
+from .tool_loader import load_tool
+
+
+def _load_skill(
+    plugin_root: Path, skills_dir: str, entry: str, item: Any
+) -> LoadedSkill:
+    if isinstance(item, str):
+        requested_name = item
+    elif isinstance(item, dict) and isinstance(item.get("name"), str):
+        if set(item) != {"name"}:
+            raise CollectionError(
+                "context.skills entries cannot inject Skill bodies; only {name} is allowed"
+            )
+        requested_name = item["name"]
+    else:
+        raise CollectionError("context.skills entries must be names or {name} objects")
+    path = (plugin_root / skills_dir / requested_name / entry).resolve()
+    if not path.is_file():
+        raise CollectionError(f"skill does not exist: {path}")
+    text = path.read_text(encoding="utf-8")
+    if not text.startswith("---\n"):
+        raise CollectionError(f"skill frontmatter is missing: {path}")
+    try:
+        _, raw_frontmatter, body = text.split("---", 2)
+        frontmatter = yaml.safe_load(raw_frontmatter)
+    except (ValueError, yaml.YAMLError) as exc:
+        raise CollectionError(f"invalid skill frontmatter: {path}") from exc
+    if not isinstance(frontmatter, dict):
+        raise CollectionError(f"invalid skill frontmatter: {path}")
+    name = frontmatter.get("name")
+    description = frontmatter.get("description")
+    if name != requested_name:
+        raise CollectionError(
+            f"skill name mismatch: requested {requested_name!r}, frontmatter has {name!r}"
+        )
+    if not isinstance(description, str) or not description.strip():
+        raise CollectionError(f"skill description is missing: {path}")
+    return LoadedSkill(
+        requested_name=requested_name,
+        name=name,
+        description=description.strip(),
+        body=body.lstrip("\n"),
+        path=path,
+        content_hash=sha256_file(path),
+    )
+
+
+def _render_system_prompt(base: str, skills: list[LoadedSkill]) -> str:
+    rendered: list[str] = []
+    for skill in skills:
+        block = (
+            "<skill>\n<name>\n"
+            f"{escape(skill.name, quote=True)}\n"
+            "</name>\n<description>\n"
+            f"{escape(skill.description, quote=True)}\n"
+            "</description>"
+        )
+        block += "\n</skill>"
+        rendered.append(block)
+    catalog = "\n".join(rendered)
+    return f"{base.rstrip()}\n<available_skills>\n{catalog}\n</available_skills>"
+
+
+_INJECT_POSITIONS = {
+    "before_user_message",
+    "after_user_message",
+    "after_all_tool_calls",
+    "after_tool_call",
+}
+
+
+def _resolve_message_inject(
+    evals_root: Path,
+    entries: Any,
+    fixture_manifest: list[dict[str, Any]],
+    step_id: str,
+) -> list[dict[str, Any]]:
+    if not isinstance(entries, list):
+        raise CollectionError(f"step {step_id} message_inject must be a list")
+    resolved: list[dict[str, Any]] = []
+    for entry in entries:
+        if not isinstance(entry, dict):
+            raise CollectionError(
+                f"step {step_id} message_inject entry must be an object"
+            )
+        unknown = set(entry) - {"position", "message", "match"}
+        if unknown:
+            raise CollectionError(
+                f"step {step_id} message_inject has unknown fields: {sorted(unknown)}"
+            )
+        position = entry.get("position")
+        if position not in _INJECT_POSITIONS:
+            raise CollectionError(
+                f"step {step_id} message_inject.position must be one of "
+                f"{sorted(_INJECT_POSITIONS)}, got {position!r}"
+            )
+        message = entry.get("message")
+        if not isinstance(message, dict) or message.get("role") not in {
+            "user",
+            "assistant",
+            "tool",
+        }:
+            raise CollectionError(
+                f"step {step_id} message_inject.message must have a valid role"
+            )
+        resolved_message = deepcopy(message)
+        resolved_message["content"] = _resolve_message_parts(
+            evals_root, message.get("content"), fixture_manifest
+        )
+        match = entry.get("match")
+        if position == "after_tool_call":
+            if not isinstance(match, dict) or not isinstance(match.get("name"), str):
+                raise CollectionError(
+                    f"step {step_id} message_inject.after_tool_call requires "
+                    "match.name"
+                )
+            match_unknown = set(match) - {"name", "arguments", "call"}
+            if match_unknown:
+                raise CollectionError(
+                    f"step {step_id} message_inject.match has unknown fields: "
+                    f"{sorted(match_unknown)}"
+                )
+            if "arguments" in match and not isinstance(match["arguments"], dict):
+                raise CollectionError(
+                    f"step {step_id} message_inject.match.arguments must be an object"
+                )
+            if "call" in match and (
+                not isinstance(match["call"], int) or match["call"] < 1
+            ):
+                raise CollectionError(
+                    f"step {step_id} message_inject.match.call must be a "
+                    "positive integer"
+                )
+        elif match is not None:
+            raise CollectionError(
+                f"step {step_id} message_inject.match is only valid for "
+                "position: after_tool_call"
+            )
+        resolved.append(
+            {
+                "position": position,
+                "message": resolved_message,
+                **({"match": deepcopy(match)} if match is not None else {}),
+            }
+        )
+    return resolved
+
+
+def _resolve_message_parts(
+    evals_root: Path,
+    content: Any,
+    fixture_manifest: list[dict[str, Any]],
+) -> Any:
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list) or not content:
+        raise CollectionError("message content must be a string or non-empty part list")
+    resolved: list[dict[str, Any]] = []
+    for part in content:
+        if not isinstance(part, dict) or part.get("type") not in {
+            "text",
+            "input_image",
+            "input_file",
+        }:
+            raise CollectionError(f"unsupported message part: {part!r}")
+        allowed_fields = (
+            {"type", "text"}
+            if part["type"] == "text"
+            else {"type", "fixture", "filename", "media_type", "delivery"}
+        )
+        unsupported_fields = set(part) - allowed_fields
+        if unsupported_fields:
+            names = ", ".join(sorted(unsupported_fields))
+            raise CollectionError(
+                f"message part contains provider-specific or unknown fields: {names}"
+            )
+        if part["type"] == "text":
+            if not isinstance(part.get("text"), str):
+                raise CollectionError("text part requires text")
+            resolved.append({"type": "text", "text": part["text"]})
+            continue
+        fixture = (evals_root / str(part.get("fixture", ""))).resolve()
+        if not fixture.is_file():
+            raise CollectionError(f"fixture does not exist: {fixture}")
+        media_type = part.get("media_type") or mimetypes.guess_type(fixture.name)[0]
+        if not media_type:
+            raise CollectionError(f"cannot infer fixture media type: {fixture}")
+        entry = {
+            "type": part["type"],
+            "fixture": str(fixture),
+            "filename": part.get("filename") or fixture.name,
+            "media_type": media_type,
+            "bytes": fixture.stat().st_size,
+            "sha256": sha256_file(fixture),
+        }
+        delivery = part.get("delivery", "file_path_inline")
+        legacy_deliveries = {
+            "metadata": "file_path_inline",
+            "content": "file_content",
+        }
+        delivery = legacy_deliveries.get(delivery, delivery)
+        if delivery not in {"file_path_inline", "file_content"}:
+            raise CollectionError(
+                "file delivery must be file_path_inline or file_content"
+            )
+        entry["delivery"] = delivery
+        resolved.append(entry)
+        if not any(
+            existing["sha256"] == entry["sha256"] for existing in fixture_manifest
+        ):
+            fixture_manifest.append(deepcopy(entry))
+    return resolved
+
+
+def resolve_case(
+    path: Path,
+    plugin_root: Path,
+    workspace: WorkspaceConfig,
+    plugin_config: dict[str, Any],
+) -> ResolvedCase:
+    raw = load_yaml(path)
+    evals_root = (plugin_root / "__evals__").resolve()
+    context = raw.get("context")
+    steps = raw.get("steps")
+    if not isinstance(context, dict):
+        raise CollectionError(f"context is required: {path}")
+    if "skill_tool" in context:
+        raise CollectionError(
+            "context.skill_tool has been removed; declare {builtin: skill} "
+            "in context.tools"
+        )
+    if not isinstance(steps, list) or not steps:
+        raise CollectionError(f"steps must be a non-empty list: {path}")
+
+    conventions = {
+        "system_prompt": plugin_config.get("system_prompt", "SYSTEM.md"),
+        "skills_dir": plugin_config.get("skills_dir", "skills"),
+        "skill_entry": plugin_config.get("skill_entry", "SKILL.md"),
+    }
+    system_config = context.get("system_prompt", {"default": True})
+    if system_config is None or system_config == {"default": True}:
+        system_path = plugin_root / conventions["system_prompt"]
+    elif isinstance(system_config, dict) and isinstance(system_config.get("file"), str):
+        system_path = evals_root / system_config["file"]
+    else:
+        raise CollectionError(
+            "context.system_prompt must use {default: true} or {file: ...}"
+        )
+    if not system_path.is_file():
+        raise CollectionError(f"system prompt does not exist: {system_path}")
+
+    skills = [
+        _load_skill(
+            plugin_root, conventions["skills_dir"], conventions["skill_entry"], item
+        )
+        for item in context.get("skills", [])
+    ]
+    names = [skill.name for skill in skills]
+    if len(names) != len(set(names)):
+        raise CollectionError("context.skills contains duplicate names")
+    rendered_system = _render_system_prompt(
+        system_path.read_text(encoding="utf-8"), skills
+    )
+
+    mocks = deepcopy(raw.get("mocks", {}))
+    if not isinstance(mocks, dict):
+        raise CollectionError("mocks must be an object")
+    tool_items = context.get("tools", [])
+    if not isinstance(tool_items, list):
+        raise CollectionError("context.tools must be a list")
+    tools = [
+        load_tool(
+            evals_root,
+            item,
+            skills=skills,
+            conventions=conventions,
+            mocks=mocks,
+        )
+        for item in tool_items
+    ]
+    if skills and not any(
+        isinstance(item, dict) and item.get("builtin") == "skill"
+        for item in tool_items
+    ):
+        raise CollectionError(
+            "context.tools must include {builtin: skill} when context.skills "
+            "is non-empty"
+        )
+    tool_names = [tool.name for tool in tools]
+    if len(tool_names) != len(set(tool_names)):
+        raise CollectionError("context.tools contains duplicate names")
+    if "cli" in mocks:
+        raise CollectionError(
+            "mocks.cli requires context.tools to include {builtin: bash}"
+        )
+    if "fs" in mocks:
+        raise CollectionError(
+            "mocks.fs requires context.tools to include {builtin: bash}"
+        )
+
+    fixture_manifest: list[dict[str, Any]] = []
+    initial_messages: list[dict[str, Any]] = [
+        {"role": "system", "content": rendered_system}
+    ]
+    for message in context.get("messages", []):
+        if not isinstance(message, dict) or message.get("role") not in {
+            "user",
+            "assistant",
+            "tool",
+        }:
+            raise CollectionError(f"invalid context message: {message!r}")
+        resolved = deepcopy(message)
+        resolved["content"] = _resolve_message_parts(
+            evals_root, message.get("content"), fixture_manifest
+        )
+        initial_messages.append(resolved)
+
+    resolved_steps: list[dict[str, Any]] = []
+    step_ids: set[str] = set()
+    for index, step in enumerate(steps):
+        if not isinstance(step, dict) or not isinstance(step.get("id"), str):
+            raise CollectionError(f"step {index + 1} requires an id")
+        if step["id"] in step_ids:
+            raise CollectionError(f"duplicate step id: {step['id']}")
+        step_ids.add(step["id"])
+        if "user" not in step:
+            raise CollectionError(f"step requires user: {step['id']}")
+        resolved = deepcopy(step)
+        resolved["user"] = _resolve_message_parts(
+            evals_root, step["user"], fixture_manifest
+        )
+        if "message_inject" in step:
+            resolved["message_inject"] = _resolve_message_inject(
+                evals_root, step["message_inject"], fixture_manifest, step["id"]
+            )
+        resolved["index"] = index
+        resolved_steps.append(resolved)
+
+    model = deepcopy(workspace.defaults["model"])
+    model.update(plugin_config.get("model", {}))
+    model.update(raw.get("model", {}))
+    execution = deepcopy(workspace.defaults["execution"])
+    execution.update(plugin_config.get("execution", {}))
+    execution.update(raw.get("execution", {}))
+    try:
+        case_key = path.resolve().relative_to(plugin_root.resolve()).as_posix()
+    except ValueError:
+        case_key = path.name
+    return ResolvedCase(
+        path=path.resolve(),
+        case_key=case_key,
+        plugin_root=plugin_root.resolve(),
+        evals_root=evals_root,
+        model=model,
+        context=context,
+        mocks=mocks,
+        execution=execution,
+        steps=resolved_steps,
+        outcome=deepcopy(raw.get("outcome", {})),
+        system_prompt=rendered_system,
+        skills=skills,
+        tools=tools,
+        initial_messages=initial_messages,
+        fixture_manifest=fixture_manifest,
+    )

yama/loaders/common.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+class CollectionError(ValueError):
+    pass
+
+
+def load_yaml(path: Path) -> dict[str, Any]:
+    try:
+        value = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except (OSError, yaml.YAMLError) as exc:
+        raise CollectionError(f"cannot load YAML {path}: {exc}") from exc
+    if not isinstance(value, dict):
+        raise CollectionError(f"YAML root must be an object: {path}")
+    return value
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()

yama/loaders/tool_loader.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from ..mocks import MockConfigurationError
+from ..models import LoadedSkill, ToolSchema
+from ..tools.bash.bash import normalize_cli_config
+from ..tools.bash.fs import normalize_fs_config
+from .common import CollectionError, load_yaml
+
+
+def load_tool(
+    evals_root: Path,
+    item: Any,
+    *,
+    skills: list[LoadedSkill],
+    conventions: dict[str, str],
+    mocks: dict[str, Any],
+) -> ToolSchema:
+    if not isinstance(item, dict) or sum(
+        key in item for key in ("builtin", "file", "inline")
+    ) != 1:
+        raise CollectionError(
+            "each context.tools entry must choose exactly one of builtin, file or inline"
+        )
+    if "builtin" in item:
+        if set(item) != {"builtin"}:
+            raise CollectionError("builtin tool entries cannot contain other fields")
+        if item["builtin"] == "skill":
+            return _resolve_builtin_skill_tool(skills, conventions, mocks)
+        if item["builtin"] == "bash":
+            return _resolve_builtin_bash_tool(mocks, evals_root)
+        raise CollectionError(f"unknown builtin tool: {item['builtin']!r}")
+    source_path: Path | None = None
+    if "file" in item:
+        source_path = (evals_root / str(item["file"])).resolve()
+        if source_path.suffix not in {".yaml", ".yml"}:
+            raise CollectionError(f"tool schema must be YAML: {source_path}")
+        raw = load_yaml(source_path)
+    else:
+        raw = item["inline"]
+    if not isinstance(raw, dict):
+        raise CollectionError("inline tool schema must be an object")
+    name = raw.get("name")
+    description = raw.get("description")
+    parameters = raw.get("parameters")
+    if not isinstance(name, str) or not name:
+        raise CollectionError("tool name is required")
+    if not isinstance(description, str) or not description:
+        raise CollectionError(f"tool description is required: {name}")
+    if not isinstance(parameters, dict) or parameters.get("type") != "object":
+        raise CollectionError(f"tool parameters root must have type=object: {name}")
+    return ToolSchema(name, description, parameters, source_path)
+
+
+def _skill_tool_schema(skills: list[LoadedSkill]) -> ToolSchema:
+    names = [skill.name for skill in skills]
+    return ToolSchema(
+        name="skill",
+        description="Read an enabled Skill entry or one file inside that Skill directory.",
+        parameters={
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string",
+                    "enum": names,
+                    "description": "Enabled Skill name.",
+                },
+                "file": {
+                    "type": "string",
+                    "description": "Optional path relative to the selected Skill directory.",
+                },
+            },
+            "required": ["name"],
+            "additionalProperties": False,
+        },
+    )
+
+
+def _resolve_builtin_skill_tool(
+    skills: list[LoadedSkill],
+    conventions: dict[str, str],
+    mocks: dict[str, Any],
+) -> ToolSchema:
+    if not skills:
+        raise CollectionError(
+            "context.tools builtin skill requires at least one context.skills entry"
+        )
+
+    mock_tools = mocks.setdefault("tools", {})
+    if not isinstance(mock_tools, dict):
+        raise CollectionError("mocks.tools must be an object")
+    if "skill" not in mock_tools:
+        mock_tools["skill"] = {
+            "match": {
+                "arguments": {
+                    "name": {"one_of": [skill.name for skill in skills]},
+                }
+            },
+            "handler": {
+                "python": "yama.tools.skill.skill:read_skill",
+                "with": {
+                    "enabled": [skill.name for skill in skills],
+                    "skills_dir": conventions["skills_dir"],
+                    "skill_entry": conventions["skill_entry"],
+                },
+            },
+        }
+    return _skill_tool_schema(skills)
+
+
+def _bash_tool_schema() -> ToolSchema:
+    return ToolSchema(
+        name="bash",
+        description=(
+            "Execute a bash command and return its stdout, stderr and exit code."
+        ),
+        parameters={
+            "type": "object",
+            "properties": {
+                "command": {
+                    "type": "string",
+                    "description": "The bash command to execute.",
+                },
+            },
+            "required": ["command"],
+            "additionalProperties": False,
+        },
+    )
+
+
+def _resolve_builtin_bash_tool(mocks: dict[str, Any], evals_root: Path) -> ToolSchema:
+    mock_tools = mocks.setdefault("tools", {})
+    if not isinstance(mock_tools, dict):
+        raise CollectionError("mocks.tools must be an object")
+    cli_raw = mocks.pop("cli", {})
+    fs_raw = mocks.pop("fs", None)
+    if cli_raw and fs_raw is not None:
+        raise CollectionError("mocks.cli cannot be combined with mocks.fs")
+    try:
+        cli = normalize_cli_config(cli_raw, base_dir=evals_root, root=evals_root.parent)
+    except MockConfigurationError as exc:
+        raise CollectionError(str(exc)) from exc
+    fs_normalized: dict[str, Any] | None = None
+    if fs_raw is not None:
+        try:
+            fs_normalized = normalize_fs_config(
+                fs_raw, base_dir=evals_root, root=evals_root.parent
+            )
+        except MockConfigurationError as exc:
+            raise CollectionError(str(exc)) from exc
+    if "bash" in mock_tools:
+        # Check the raw pre-normalization values: normalize_cli_config always
+        # returns a non-empty dict (passthrough defaults), so testing the
+        # normalized `cli` here would flag every custom handler as conflicting
+        # even when the case never declared mocks.cli/mocks.fs at all.
+        if cli_raw or fs_raw is not None:
+            raise CollectionError(
+                "mocks.cli/mocks.fs cannot be combined with a custom "
+                "mocks.tools.bash handler"
+            )
+    else:
+        handler_with: dict[str, Any] = (
+            {"fs": fs_normalized} if fs_normalized is not None else {"cli": cli}
+        )
+        mock_tools["bash"] = {
+            "handler": {
+                "python": "yama.tools.bash.bash:run_bash",
+                "with": handler_with,
+            },
+        }
+    return _bash_tool_schema()