pytest-skill-engineering 1.0.2__tar.gz → 1.0.3__tar.gz

{pytest_skill_engineering-1.0.2 → pytest_skill_engineering-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pytest-skill-engineering
-Version: 1.0.2
+Version: 1.0.3
 Summary: Reliable Copilot execution and evidence for coding-agent-led testing of MCP servers, CLI tools, skills, and custom agents.
 Project-URL: Homepage, https://github.com/sbroenne/pytest-skill-engineering
 Project-URL: Repository, https://github.com/sbroenne/pytest-skill-engineering

{pytest_skill_engineering-1.0.2 → pytest_skill_engineering-1.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pytest-skill-engineering"
-version = "1.0.2"
+version = "1.0.3"
 description = "Reliable Copilot execution and evidence for coding-agent-led testing of MCP servers, CLI tools, skills, and custom agents."
 readme = "README.md"
 license = { text = "MIT" }

pytest_skill_engineering-1.0.3/src/pytest_skill_engineering/copilot/config.py

@@ -0,0 +1,133 @@
+"""Configuration file loaders for Copilot and Claude Code projects.
+
+Provides utilities for loading MCP server configurations from standard
+config files (``.mcp.json``, ``.vscode/mcp.json``).
+"""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from pytest_skill_engineering.core.plugin import _read_json
+from pytest_skill_engineering.core.validation import _validate_mcp_servers
+
+if TYPE_CHECKING:
+    from pytest_skill_engineering.copilot.contracts import CopilotEvalConfig
+
+
+def snapshot_session_configuration(
+    agent: CopilotEvalConfig, session_config: dict[str, Any]
+) -> dict[str, Any]:
+    """Capture prepared settings without handlers, credentials, or server environments."""
+    system = session_config.get("system_message") or {}
+    snapshot = {
+        "name": agent.name,
+        "model": session_config.get("model"),
+        "instructions": system.get("content"),
+        "system_message_mode": system.get("mode"),
+        "reasoning_effort": session_config.get("reasoning_effort"),
+        "client_mode": agent.client_mode,
+        "image_detail": agent.image_detail,
+        "audit_requests": agent.audit_requests,
+        "max_tool_calls": agent.max_tool_calls,
+        "max_turns": agent.max_turns,
+        "timeout_s": agent.timeout_s,
+        "auto_confirm": agent.auto_confirm,
+        "allowed_tools": session_config.get("available_tools"),
+        "excluded_tools": session_config.get("excluded_tools"),
+        "skill_directories": session_config.get("skill_directories", []),
+        "disabled_skills": session_config.get("disabled_skills", []),
+        "active_agent": session_config.get("agent"),
+        "working_directory": session_config.get("working_directory"),
+        "persona": type(agent.persona).__name__,
+        "tools": [
+            {
+                "name": tool.name,
+                "description": tool.description,
+                "parameters": tool.parameters,
+                "metadata": tool.metadata,
+            }
+            for tool in session_config.get("tools") or []
+        ],
+        "mcp_servers": {
+            name: {"type": config.get("type"), "tools": config.get("tools")}
+            for name, config in (session_config.get("mcp_servers") or {}).items()
+        },
+        "custom_agents": [
+            {
+                key: value
+                for key, value in custom.items()
+                if key
+                in {
+                    "name",
+                    "prompt",
+                    "description",
+                    "display_name",
+                    "model",
+                    "reasoning_effort",
+                    "tools",
+                    "infer",
+                    "skills",
+                }
+            }
+            for custom in session_config.get("custom_agents") or []
+        ],
+    }
+    for key in (
+        "enable_config_discovery",
+        "enable_file_hooks",
+        "enable_on_demand_instruction_discovery",
+        "skip_custom_instructions",
+        "enable_managed_settings",
+        "enable_skills",
+    ):
+        if key in session_config:
+            snapshot[key] = session_config[key]
+    return deepcopy(snapshot)
+
+
+def load_mcp_config(path: str | Path) -> dict[str, dict[str, Any]]:
+    """Load MCP server configs from a ``.mcp.json`` file.
+
+    Supports both Claude Code / root-level ``.mcp.json`` and VS Code
+    ``.vscode/mcp.json`` formats.  Handles two common top-level keys:
+
+    * ``mcpServers`` — Claude Code / standard MCP convention
+    * ``servers`` — VS Code ``mcp.json`` convention
+
+    Exactly one supported key must be present. An explicit empty mapping is valid;
+    missing keys, conflicting keys, and malformed server definitions raise errors.
+
+    Returns:
+        Dict of ``server_name → config`` compatible with
+        ``CopilotEval(mcp_servers=...)``.
+
+    Raises:
+        FileNotFoundError: If the file does not exist.
+        ValueError: If the file is not valid JSON or has unexpected structure.
+
+    Example::
+
+        from pytest_skill_engineering.copilot.config import load_mcp_config
+
+        servers = load_mcp_config(".mcp.json")
+        agent = CopilotEval(mcp_servers=servers)
+    """
+    path = Path(path)
+    if not path.is_file():
+        msg = f"MCP config file not found: {path}"
+        raise FileNotFoundError(msg)
+
+    raw = _read_json(path)
+
+    if not isinstance(raw, dict):
+        msg = f"{path}: MCP config must be a JSON object, got {type(raw).__name__}"
+        raise ValueError(msg)
+
+    fields = [key for key in ("mcpServers", "servers") if key in raw]
+    if len(fields) != 1:
+        raise ValueError(f"{path}: specify exactly one of 'mcpServers' and 'servers'")
+
+    return _validate_mcp_servers(raw[fields[0]], path, field=fields[0])

{pytest_skill_engineering-1.0.2 → pytest_skill_engineering-1.0.3}/src/pytest_skill_engineering/copilot/contracts.py

@@ -3,24 +3,16 @@
 from __future__ import annotations
 
 from collections.abc import Awaitable, Callable, Mapping
-from dataclasses import dataclass
 from typing import Any, Literal, Protocol, TypeAlias
 
 from copilot.session import CustomAgentConfig, MCPServerConfig, ReasoningEffort, SessionHooks
 
+from pytest_skill_engineering.copilot.result import CopilotResult, CopilotResultAgent
+
 CopilotCustomAgentConfig: TypeAlias = CustomAgentConfig
 CopilotMCPServerConfig: TypeAlias = MCPServerConfig
 CopilotReasoningEffort: TypeAlias = ReasoningEffort
 CopilotSessionHooks: TypeAlias = SessionHooks
-SubagentStatus: TypeAlias = Literal["selected", "started", "completed", "failed"]
-
-
-class CopilotResultAgent(Protocol):
-    """Minimal agent surface stored on :class:`CopilotResult`."""
-
-    @property
-    def working_directory(self) -> str | None:
-        raise NotImplementedError
 
 
 class CopilotRunResult(Protocol):
@@ -53,6 +45,11 @@ class CopilotEventMapper(Protocol):
     def record_subagent_failed(self, *, invocation_id: str, name: str) -> None:
         raise NotImplementedError
 
+    def record_subagent_result(
+        self, *, invocation_id: str, name: str, result: CopilotResult
+    ) -> None:
+        raise NotImplementedError
+
 
 class CopilotEvalConfig(CopilotResultAgent, Protocol):
     """Configuration surface consumed by the runner and personas."""
@@ -141,7 +138,7 @@ class CopilotEvalConfig(CopilotResultAgent, Protocol):
 
 CopilotNestedRunner: TypeAlias = Callable[
     [CopilotEvalConfig, str],
-    Awaitable[CopilotRunResult],
+    Awaitable["CopilotResult"],
 ]
 
 
@@ -158,16 +155,6 @@ class CopilotPersona(Protocol):
         raise NotImplementedError
 
 
-@dataclass(slots=True)
-class SubagentInvocation:
-    """A single custom-agent dispatch observed during a run."""
-
-    invocation_id: str
-    name: str
-    status: SubagentStatus
-    duration_ms: float | None = None
-
-
 def declared_agent_tools(
     agent_config: CopilotCustomAgentConfig,
 ) -> list[str] | None:

{pytest_skill_engineering-1.0.2 → pytest_skill_engineering-1.0.3}/src/pytest_skill_engineering/copilot/eval.py

@@ -18,7 +18,6 @@ from pytest_skill_engineering.copilot.personas import ClaudeCodePersona, VSCodeP
 from pytest_skill_engineering.core.evals import load_custom_agent, load_instruction_file
 from pytest_skill_engineering.core.plugin import (
     _discover_skills,
-    _validate_mcp_servers,
     _validate_skill_reference_names,
 )
 
@@ -34,30 +33,20 @@ def _parse_agent_file(path: Path) -> CopilotCustomAgentConfig:
     fields and uses the Markdown body as the custom agent's system prompt.
     Parsing and metadata errors retain the definition's file path.
     """
-    loaded = load_custom_agent(path)
-    frontmatter = loaded["metadata"]
-    agent: CopilotCustomAgentConfig = {
-        "name": frontmatter.get("name", loaded["name"]),
-        "prompt": loaded["prompt"],
-    }
-    for key in (
-        "description",
-        "display_name",
-        "infer",
-        "skills",
-        "model",
-        "reasoning_effort",
-        "tools",
-    ):
-        if key in frontmatter:
-            agent[key] = frontmatter[key]
-    if "mcp-servers" in frontmatter:
-        agent["mcp_servers"] = cast(
-            dict[str, CopilotMCPServerConfig],
-            _validate_mcp_servers(frontmatter["mcp-servers"], path, field="mcp-servers"),
-        )
+    return cast(CopilotCustomAgentConfig, load_custom_agent(path))
+
 
-    return agent
+def _parse_agent_files(paths: list[Path]) -> list[CopilotCustomAgentConfig]:
+    agents: list[CopilotCustomAgentConfig] = []
+    names: set[str] = set()
+    for path in paths:
+        agent = _parse_agent_file(path)
+        name = require_custom_agent_name(agent)
+        if name in names:
+            raise ValueError(f"{path}: Duplicate custom agent name '{name}'")
+        names.add(name)
+        agents.append(agent)
+    return agents
 
 
 def _default_persona() -> CopilotPersona:
@@ -361,8 +350,7 @@ class CopilotEval:
         if agents_dir.exists():
             if not agents_dir.is_dir():
                 raise ValueError(f"{agents_dir}: custom agents path must be a directory")
-            for agent_file in sorted(agents_dir.rglob("*.agent.md")):
-                agents.append(_parse_agent_file(agent_file))
+            agents = _parse_agent_files(sorted(agents_dir.rglob("*.agent.md")))
 
         config: dict[str, Any] = {
             "instructions": instructions,
@@ -527,8 +515,7 @@ class CopilotEval:
             if not agents_dir.is_dir():
                 raise ValueError(f"{agents_dir}: custom agents path must be a directory")
             # Claude Code uses plain .md files for agents
-            for agent_file in sorted(agents_dir.glob("*.md")):
-                agents.append(_parse_agent_file(agent_file))
+            agents = _parse_agent_files(sorted(agents_dir.glob("*.md")))
 
         # 3. Discover skill directories from .claude/skills/
         skills = _discover_skills(claude_dir)

{pytest_skill_engineering-1.0.2 → pytest_skill_engineering-1.0.3}/src/pytest_skill_engineering/copilot/events.py

@@ -56,20 +56,23 @@ SDK Event Types (38 values) grouped by what they map to:
 
 from __future__ import annotations
 
+import base64
+import binascii
 import logging
 import time
 from collections import deque
 from datetime import timedelta
 from typing import TYPE_CHECKING, Any
 
-from pytest_skill_engineering.copilot.contracts import SubagentStatus
 from pytest_skill_engineering.copilot.result import (
     CopilotResult,
     SubagentInvocation,
+    SubagentStatus,
     ToolCall,
     Turn,
     UsageInfo,
 )
+from pytest_skill_engineering.core.result import ImageContent
 
 if TYPE_CHECKING:
     from copilot.generated.session_events import SessionEvent
@@ -162,7 +165,12 @@ class EventMapper:
             permissions=self._permissions,
             model_used=self._model_used,
             raw_events=self._raw_events,
-            total_premium_requests=self._total_premium_requests,
+            total_premium_requests=self._total_premium_requests
+            + sum(
+                invocation.result.total_premium_requests
+                for invocation in resolved_subagents
+                if invocation.result is not None
+            ),
         )
 
     # ── Assistant events ──
@@ -342,22 +350,73 @@ class EventMapper:
             self._early_tool_completions.setdefault(call_id, []).append(event)
             return
         result_text: str | None = None
+        images: list[ImageContent] = []
         if result_data is not None and hasattr(result_data, "content"):
             result_text = result_data.content
         elif isinstance(result_data, str):
             result_text = result_data
         elif result_data is not None:
             result_text = str(result_data)
+        text_parts: list[str] = []
+        blocks = getattr(result_data, "contents", None) or []
+        for block in blocks:
+            block_type = getattr(block, "type", None)
+            if block_type == "image":
+                try:
+                    data = getattr(block, "data", None)
+                    media_type = getattr(block, "mime_type", None)
+                    if (
+                        not isinstance(data, str)
+                        or not isinstance(media_type, str)
+                        or not media_type
+                    ):
+                        raise ValueError("image data and media type are required")
+                    images.append(ImageContent(base64.b64decode(data, validate=True), media_type))
+                except (ValueError, binascii.Error) as exc:
+                    self._record_contract_error(f"Tool call {call_id} has an invalid image: {exc}")
+            elif block_type == "text" and result_text is None:
+                text = getattr(block, "text", None)
+                if text is not None:
+                    text_parts.append(text)
+        if result_text is None and text_parts:
+            result_text = "\n".join(text_parts)
+        content_images = list(images)
+        for block in getattr(result_data, "binary_results_for_llm", None) or []:
+            block_type = getattr(block, "type", None)
+            if getattr(block_type, "value", block_type) != "image":
+                continue
+            try:
+                data, media_type = block.data, block.mime_type
+                if not isinstance(data, str) or not isinstance(media_type, str) or not media_type:
+                    raise ValueError("image data and media type are required")
+                image = ImageContent(base64.b64decode(data, validate=True), media_type)
+                if image not in content_images:
+                    images.append(image)
+            except (ValueError, binascii.Error) as exc:
+                self._record_contract_error(f"Tool call {call_id} has an invalid image: {exc}")
         success = _get_data_field(event, "success", None)
         error = _stringify_tool_error(_get_data_field(event, "error", None))
         if tc.completion_received:
-            if (tc.result, tc.success, tc.error) != (result_text, success, error):
+            captured_images = (
+                [ImageContent(tc.image_content, tc.image_media_type or "image/png")]
+                if tc.image_content is not None
+                else []
+            ) + tc.additional_images
+            if (tc.result, tc.success, tc.error, captured_images) != (
+                result_text,
+                success,
+                error,
+                images,
+            ):
                 self._record_contract_error(f"Conflicting completion for tool call {call_id}")
             return
         tc.completion_received = True
         tc.success = success
         tc.result = result_text
         tc.error = error
+        if images:
+            tc.image_content, tc.image_media_type = images[0].data, images[0].media_type
+            tc.additional_images = images[1:]
         if tc.success is None:
             self._record_contract_error(f"Tool call {call_id} completion is missing success")
         start = self._pending_tool_start_times.pop(call_id, None)
@@ -382,6 +441,16 @@ class EventMapper:
 
     # ── Subagent recording (used by runSubagent tool handler) ──
 
+    def record_subagent_result(
+        self, *, invocation_id: str, name: str, result: CopilotResult
+    ) -> None:
+        """Link a framework-owned child without mixing its conversation into this one."""
+        invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)
+        if invocation.result is not None and invocation.result is not result:
+            self._record_contract_error(f"Conflicting child result for invocation {invocation_id}")
+            return
+        invocation.result = result
+
     def record_subagent_start(self, *, invocation_id: str, name: str) -> None:
         """Record a subagent invocation dispatched via a tool call."""
         invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)

{pytest_skill_engineering-1.0.2 → pytest_skill_engineering-1.0.3}/src/pytest_skill_engineering/copilot/fixtures.py

@@ -83,7 +83,7 @@ def _convert_to_aitest(
     """
     from dataclasses import dataclass
 
-    from pytest_skill_engineering.core.result import EvalResult
+    from pytest_skill_engineering.core.result import EvalResult, SubagentInvocation
     from pytest_skill_engineering.execution.cost import estimate_cost
 
     # Estimate USD cost from captured token usage and pricing.toml.
@@ -96,7 +96,7 @@ def _convert_to_aitest(
             usage.output_tokens,
             usage.cache_read_tokens or 0,
         )
-        for usage in result.usage
+        for usage in result.all_usage
         if usage.input_tokens is not None and usage.output_tokens is not None
     )
 
@@ -108,7 +108,9 @@ def _convert_to_aitest(
         duration_ms=result.duration_ms,
         token_usage=result.token_usage,
         cost_usd=cost_usd,
-        effective_system_prompt=agent.instructions or "",
+        effective_system_prompt=(
+            result.configuration.get("instructions") if result.configuration is not None else None
+        ),
         premium_requests=result.total_premium_requests,
         evidence_complete=result.evidence_complete,
         capture_errors=list(result.capture_errors),
@@ -117,49 +119,39 @@ def _convert_to_aitest(
         usage=list(result.usage),
         tool_calls_admitted=result.tool_calls_admitted,
         skill_discovery=deepcopy(result.skill_discovery),
-        configuration=deepcopy(
-            {
-                "name": agent.name,
-                "model": agent.model,
-                "instructions": agent.instructions,
-                "system_message_mode": agent.system_message_mode,
-                "reasoning_effort": agent.reasoning_effort,
-                "client_mode": agent.client_mode,
-                "image_detail": agent.image_detail,
-                "audit_requests": agent.audit_requests,
-                "max_tool_calls": agent.max_tool_calls,
-                "tools": [
-                    {
-                        "name": tool.name,
-                        "description": tool.description,
-                        "parameters": tool.parameters,
-                        "metadata": tool.metadata,
-                    }
-                    for tool in agent.extra_config.get("tools", [])
-                ],
-                "allowed_tools": agent.allowed_tools,
-                "excluded_tools": agent.excluded_tools,
-                "max_turns": agent.max_turns,
-                "timeout_s": agent.timeout_s,
-                "auto_confirm": agent.auto_confirm,
-                "mcp_servers": {
-                    name: {"type": config.get("type"), "tools": config.get("tools")}
-                    for name, config in agent.mcp_servers.items()
-                },
-                "skill_directories": agent.skill_directories,
-                "disabled_skills": agent.disabled_skills,
-                "active_agent": agent.active_agent,
-                "persona": type(agent.persona).__name__,
-            }
-        ),
+        configuration=deepcopy(result.configuration),
+        model_used=result.model_used,
+        reasoning_traces=list(result.reasoning_traces),
+        permission_requested=result.permission_requested,
+        permissions=deepcopy(result.permissions),
     )
+    for invocation in result.subagent_invocations:
+        child_result = None
+        if invocation.result is not None:
+            from pytest_skill_engineering.copilot.eval import CopilotEval
+
+            child_agent = invocation.result.agent
+            if not isinstance(child_agent, CopilotEval):
+                raise ValueError(f"Child invocation {invocation.invocation_id!r} has no eval")
+            converted_child = _convert_to_aitest(child_agent, invocation.result)
+            assert converted_child is not None
+            child_result = converted_child[0]
+        aitest_result.subagent_invocations.append(
+            SubagentInvocation(
+                invocation_id=invocation.invocation_id,
+                name=invocation.name,
+                status=invocation.status,
+                duration_ms=invocation.duration_ms,
+                result=child_result,
+            )
+        )
 
     # Create a minimal wrapper with just the fields needed by plugin.py
-    @dataclass
+    @dataclass(slots=True)
     class Provider:
-        model: str
+        model: str | None
 
-    @dataclass
+    @dataclass(slots=True)
     class AgentWrapper:
         name: str
         id: str
@@ -176,7 +168,15 @@ def _convert_to_aitest(
     aitest_agent = AgentWrapper(
         name=agent.name,
         id=agent.name,
-        provider=Provider(model=result.model_used or agent.model or "claude-haiku-4-5"),
+        provider=Provider(
+            model=result.model_used
+            if result.model_used is not None
+            else (
+                result.configuration.get("model")
+                if result.configuration is not None
+                else agent.extra_config.get("model", agent.model)
+            )
+        ),
         system_prompt_name=None,  # CopilotEval doesn't have named prompts
         mcp_servers=[],  # Could convert agent.mcp_servers if needed
         allowed_tools=agent.allowed_tools,
@@ -211,7 +211,7 @@ def stash_on_item(
             report_name = f"{agent.name} ({comparison_role})"
             converted[1].name = report_name
             converted[1].id = report_name
-            converted[0].configuration["comparison_role"] = comparison_role
+            converted[0].comparison_role = comparison_role
         item._aitest_result = converted[0]  # type: ignore[attr-defined]
         item._aitest_agent = converted[1]  # type: ignore[attr-defined]
         runs = getattr(item, "_aitest_runs", None)

{pytest_skill_engineering-1.0.2 → pytest_skill_engineering-1.0.3}/src/pytest_skill_engineering/copilot/personas.py

@@ -437,6 +437,10 @@ def _make_subagent_dispatch_tool(
         )
 
         sub_result = await nested_runner(sub_agent, prompt_text)
+        sub_result.agent = sub_agent
+        mapper.record_subagent_result(
+            invocation_id=invocation.tool_call_id, name=agent_slug, result=sub_result
+        )
 
         if sub_result.success:
             mapper.record_subagent_complete(
@@ -501,32 +505,26 @@ def _inject_skill_reference_tools(
 
     Only injected when at least one skill directory has reference documents.
     """
-    if not agent.skill_directories:
-        return
+    from pytest_skill_engineering.copilot.skills import requested_skill_files
+    from pytest_skill_engineering.core.plugin import _validate_skill_reference_names
+    from pytest_skill_engineering.core.skill import Skill
 
-    # Collect all reference files across all skill directories
-    reference_files: dict[str, Path] = {}  # filename → full path
-
-    for skill_dir_str in agent.skill_directories:
-        skill_dir = Path(skill_dir_str)
-
-        # Check if this is a skill directory (contains SKILL.md)
-        _EXTS = (".md", ".txt", ".json", ".yaml", ".yml")
-        if (skill_dir / "SKILL.md").exists():
-            refs_dir = skill_dir / "references"
-            if refs_dir.is_dir():
-                for ref_file in refs_dir.iterdir():
-                    if ref_file.is_file() and ref_file.suffix in _EXTS:
-                        reference_files[ref_file.name] = ref_file
-        else:
-            # Maybe it's a parent directory containing skill subdirectories
-            for sub in skill_dir.iterdir():
-                if sub.is_dir() and (sub / "SKILL.md").exists():
-                    refs_dir = sub / "references"
-                    if refs_dir.is_dir():
-                        for ref_file in refs_dir.iterdir():
-                            if ref_file.is_file() and ref_file.suffix in _EXTS:
-                                reference_files[ref_file.name] = ref_file
+    configured = (
+        session_config if "skill_directories" in session_config else agent.build_session_config()
+    )
+    directories = configured.get("skill_directories")
+    if not directories:
+        return
+    disabled = configured.get("disabled_skills") or []
+    skills = [
+        Skill.from_path(path)
+        for path, name in requested_skill_files(directories).items()
+        if name not in disabled
+    ]
+    _validate_skill_reference_names(skills)
+    reference_files = {
+        name: skill.path / "references" / name for skill in skills for name in skill.references
+    }
 
     if not reference_files:
         return