pytest-skill-engineering 1.0.0__tar.gz → 1.0.1__tar.gz

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pytest-skill-engineering
-Version: 1.0.0
+Version: 1.0.1
 Summary: Reliable Copilot execution and evidence for coding-agent-led testing of MCP servers, CLI tools, skills, and custom agents.
 Project-URL: Homepage, https://github.com/sbroenne/pytest-skill-engineering
 Project-URL: Repository, https://github.com/sbroenne/pytest-skill-engineering

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pytest-skill-engineering"
-version = "1.0.0"
+version = "1.0.1"
 description = "Reliable Copilot execution and evidence for coding-agent-led testing of MCP servers, CLI tools, skills, and custom agents."
 readme = "README.md"
 license = { text = "MIT" }

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/src/pytest_skill_engineering/copilot/__init__.py

@@ -22,11 +22,14 @@ from pytest_skill_engineering.copilot.personas import (
 from pytest_skill_engineering.copilot.requests import RequestAudit
 from pytest_skill_engineering.copilot.result import CopilotResult, UsageInfo
 from pytest_skill_engineering.core.evals import load_custom_agent, load_custom_agents
+from pytest_skill_engineering.core.result import DiscoveredSkill, SkillDiscovery
 
 __all__ = [
     "CopilotEval",
     "CopilotResult",
+    "DiscoveredSkill",
     "RequestAudit",
+    "SkillDiscovery",
     "UsageInfo",
     "ClaudeCodePersona",
     "CopilotCLIPersona",

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/src/pytest_skill_engineering/copilot/eval.py

@@ -139,7 +139,7 @@ class CopilotEval:
     # display_name, tools, mcp_servers, infer)
     custom_agents: list[CopilotCustomAgentConfig] = field(default_factory=list)
 
-    # Skill directories
+    # Explicit skills are enabled and checked before model execution.
     skill_directories: list[str] = field(default_factory=list)
     disabled_skills: list[str] = field(default_factory=list)
 
@@ -235,8 +235,24 @@ class CopilotEval:
             config["hooks"] = self.hooks
 
         # Apply extra_config passthrough
+        if (
+            self.skill_directories
+            and "skill_directories" in self.extra_config
+            and self.extra_config["skill_directories"] != self.skill_directories
+        ):
+            raise ValueError("extra_config must not override explicitly supplied skill_directories")
         config.update(self.extra_config)
 
+        if config.get("skill_directories"):
+            if config.get("enable_skills") is False:
+                raise ValueError(
+                    "Explicit skill_directories conflict with enable_skills=False; "
+                    "remove the directories for a no-skill baseline, or enable skills"
+                )
+            if config.get("enable_skills") is not None and config["enable_skills"] is not True:
+                raise ValueError("enable_skills must be a boolean or None")
+            config["enable_skills"] = True
+
         if self.client_mode == "empty":
             isolated = {
                 "enable_config_discovery": False,

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/src/pytest_skill_engineering/copilot/fixtures.py

@@ -116,6 +116,7 @@ def _convert_to_aitest(
         stop_reason=result.stop_reason,
         usage=list(result.usage),
         tool_calls_admitted=result.tool_calls_admitted,
+        skill_discovery=deepcopy(result.skill_discovery),
         configuration=deepcopy(
             {
                 "name": agent.name,

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/src/pytest_skill_engineering/copilot/result.py

@@ -12,7 +12,13 @@ from typing import Any
 
 from pytest_skill_engineering.copilot.contracts import CopilotResultAgent, SubagentInvocation
 from pytest_skill_engineering.copilot.requests import RequestAudit
-from pytest_skill_engineering.core.result import StopReason, ToolCall, Turn, UsageInfo
+from pytest_skill_engineering.core.result import (
+    SkillDiscovery,
+    StopReason,
+    ToolCall,
+    Turn,
+    UsageInfo,
+)
 
 __all__ = [
     "CopilotResult",
@@ -68,6 +74,7 @@ class CopilotResult:
     request_audit: list[RequestAudit] = field(default_factory=list)
     stop_reason: StopReason | None = None
     tool_calls_admitted: int = 0
+    skill_discovery: SkillDiscovery | None = None
 
     @property
     def evidence_complete(self) -> bool:

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/src/pytest_skill_engineering/copilot/runner.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import TYPE_CHECKING
 
@@ -27,6 +28,8 @@ from pytest_skill_engineering.copilot.controls import RunControls
 from pytest_skill_engineering.copilot.events import EventMapper
 from pytest_skill_engineering.copilot.requests import RequestAuditHandler
 from pytest_skill_engineering.copilot.result import CopilotResult, StopReason
+from pytest_skill_engineering.copilot.skills import discover_requested_skills, requested_skill_files
+from pytest_skill_engineering.core.result import SkillDiscovery
 
 if TYPE_CHECKING:
     from copilot.client import CopilotClient
@@ -80,8 +83,20 @@ async def run_copilot(agent: CopilotEvalConfig, prompt: str) -> CopilotResult:
     reason: StopReason = "completed"
     error: str | None = None
     cleanup_errors: list[str] = []
+    skill_discovery: SkillDiscovery | None = None
+    send_started = False
     try:
         session_config = agent.build_session_config()
+        skill_directories = session_config.get("skill_directories") or []
+        requested_skills: dict[Path, str] = {}
+        if skill_directories:
+            skill_discovery = SkillDiscovery()
+            if not isinstance(skill_directories, list):
+                raise ValueError("skill_directories must be a list of directory paths")
+            requested_skills = requested_skill_files(skill_directories)
+            session_config["skill_directories"] = [
+                str(Path(directory).resolve()) for directory in skill_directories
+            ]
         caller_hooks = dict(session_config.get("hooks") or {})
         session_config["hooks"] = dict(caller_hooks)
         # Empty mode must not read persona instruction files or inject tools.
@@ -123,12 +138,20 @@ async def run_copilot(agent: CopilotEvalConfig, prompt: str) -> CopilotResult:
         )
 
         async def execute() -> None:
-            nonlocal session
+            nonlocal session, send_started
             assert client is not None
             async with asyncio.timeout(min(60, agent.timeout_s)):
                 await client.start()
             async with asyncio.timeout(min(30, agent.timeout_s)):
                 session = await client.create_session(**session_config)
+                if skill_discovery is not None:
+                    await discover_requested_skills(
+                        session,
+                        requested_skills,
+                        session_config.get("disabled_skills") or [],
+                        skill_discovery,
+                    )
+            send_started = True
             await session.send_and_wait(prompt, timeout=agent.timeout_s)
 
         execution = asyncio.create_task(execute())
@@ -158,6 +181,9 @@ async def run_copilot(agent: CopilotEvalConfig, prompt: str) -> CopilotResult:
     except Exception as exc:
         logger.error("Copilot execution failed: %s", exc)
         reason, error = "execution_error", str(exc)
+        if skill_discovery is not None and not skill_discovery.complete:
+            error = f"Skill setup failed before model execution: {exc}"
+            skill_discovery.errors.append(error)
     finally:
         controls.closed = True
         for watcher in watchers:
@@ -210,12 +236,13 @@ async def run_copilot(agent: CopilotEvalConfig, prompt: str) -> CopilotResult:
 
     result = mapper.build()
     result.agent = agent
+    result.skill_discovery = skill_discovery
     result.tool_calls_admitted = controls.admitted
     if result.capture_errors:
         result.capture_errors.extend(controls.incomplete_diagnostics(abort_phase=reason))
     if audit is not None and capture_requests:
         result.request_audit = list(audit.records)
-        if audit.error or (session is not None and not audit.observed_requests):
+        if audit.error or (send_started and not audit.observed_requests):
             audit_error = audit.error or "No outbound model requests captured; audit unsupported"
             result.capture_errors.append(audit_error)
             if reason == "completed" or audit.error:

pytest_skill_engineering-1.0.1/src/pytest_skill_engineering/copilot/skills.py

@@ -0,0 +1,104 @@
+"""Validate explicit skill inputs and verify native SDK availability before sending."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pytest_skill_engineering.core.result import DiscoveredSkill, SkillDiscovery
+from pytest_skill_engineering.core.skill import Skill, SkillError
+
+if TYPE_CHECKING:
+    from copilot.session import CopilotSession
+
+logger = logging.getLogger(__name__)
+
+
+def requested_skill_files(directories: list[str]) -> dict[Path, str]:
+    """Validate individual skill directories or parents of immediate skill directories.
+
+    Relative directories are resolved from the Python caller's current directory,
+    then passed to the SDK as absolute paths.
+    """
+    requested: dict[Path, str] = {}
+    names: dict[str, Path] = {}
+    for raw_directory in directories:
+        if not isinstance(raw_directory, str) or not raw_directory.strip():
+            raise SkillError("skill_directories entries must be non-empty directory paths")
+        directory = Path(raw_directory).resolve()
+        if not directory.is_dir():
+            raise SkillError(
+                f"Requested skill directory does not exist or is not a directory: {directory}"
+            )
+        try:
+            candidates = (
+                [directory]
+                if (directory / "SKILL.md").exists()
+                else [
+                    child
+                    for child in sorted(directory.iterdir())
+                    if child.is_dir() and (child / "SKILL.md").exists()
+                ]
+            )
+            if not candidates:
+                raise SkillError(f"No SKILL.md found in requested skill directory: {directory}")
+            for candidate in candidates:
+                skill = Skill.from_path(candidate)
+                path = (skill.path / "SKILL.md").resolve()
+                if skill.name in names and names[skill.name] != path:
+                    raise SkillError(
+                        f"Duplicate requested skill name '{skill.name}': "
+                        f"{names[skill.name]} and {path}"
+                    )
+                names[skill.name] = path
+                requested[path] = skill.name
+        except OSError as exc:
+            raise SkillError(f"Cannot read requested skill directory {directory}: {exc}") from exc
+    return requested
+
+
+async def discover_requested_skills(
+    session: CopilotSession,
+    requested: dict[Path, str],
+    disabled: list[str],
+    evidence: SkillDiscovery,
+) -> None:
+    """Record SDK discovery and reject missing or unexpectedly disabled requested skills."""
+    await session.rpc.skills.ensure_loaded()
+    diagnostics = await session.rpc.skills.reload()
+    evidence.warnings = list(diagnostics.warnings)
+    evidence.errors = list(diagnostics.errors)
+    for warning in evidence.warnings:
+        logger.warning("SDK skill discovery: %s", warning)
+    listed = await session.rpc.skills.list()
+    evidence.skills = [
+        DiscoveredSkill(
+            name=skill.name,
+            description=skill.description,
+            source=skill.source.value,
+            enabled=skill.enabled,
+            user_invocable=skill.user_invocable,
+            path=skill.path,
+            plugin_name=skill.plugin_name,
+            command_name=skill.command_name,
+            argument_hint=skill.argument_hint,
+        )
+        for skill in listed.skills
+    ]
+    evidence.complete = True
+    available = {
+        (Path(skill.path).resolve(), skill.name): skill
+        for skill in evidence.skills
+        if skill.path is not None
+    }
+    for path, name in requested.items():
+        skill = available.get((path, name))
+        if skill is None:
+            evidence.errors.append(
+                f"Requested skill '{name}' was not discovered by the SDK: {path}"
+            )
+        elif not skill.enabled and name not in disabled:
+            evidence.errors.append(f"Requested skill '{name}' is unexpectedly disabled: {path}")
+    if evidence.errors:
+        raise SkillError("Skill setup failed before model execution: " + "; ".join(evidence.errors))

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/src/pytest_skill_engineering/core/__init__.py

@@ -17,10 +17,12 @@ from pytest_skill_engineering.core.prompt import (
     load_system_prompts,
 )
 from pytest_skill_engineering.core.result import (
+    DiscoveredSkill,
     EvalResult,
     ImageContent,
     MCPPrompt,
     MCPPromptArgument,
+    SkillDiscovery,
     SkillInfo,
     SubagentInvocation,
     ToolCall,
@@ -38,6 +40,7 @@ from pytest_skill_engineering.core.skill_grading import export_grading
 __all__ = [
     "AITestError",
     "EvalResult",
+    "DiscoveredSkill",
     "EngineTimeoutError",
     "HookDefinition",
     "ImageContent",
@@ -48,6 +51,7 @@ __all__ = [
     "Prompt",
     "ServerStartError",
     "Skill",
+    "SkillDiscovery",
     "SkillError",
     "SkillInfo",
     "SkillMetadata",

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/src/pytest_skill_engineering/core/result.py

@@ -181,6 +181,31 @@ class SkillInfo:
         return f"SkillInfo({self.name}{refs})"
 
 
+@dataclass(slots=True, frozen=True)
+class DiscoveredSkill:
+    """Skill metadata actually returned by the public SDK, not observed use."""
+
+    name: str
+    description: str
+    source: str
+    enabled: bool
+    user_invocable: bool
+    path: str | None = None
+    plugin_name: str | None = None
+    command_name: str | None = None
+    argument_hint: str | None = None
+
+
+@dataclass(slots=True)
+class SkillDiscovery:
+    """Pre-execution discovery evidence, including incomplete or failed checks."""
+
+    skills: list[DiscoveredSkill] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+    complete: bool = False
+
+
 @dataclass(slots=True)
 class ClarificationStats:
     """Statistics about clarification requests detected during execution.
@@ -280,6 +305,7 @@ class EvalResult:
     # Recorded interface metadata.
     available_tools: list[ToolInfo] = field(default_factory=list)
     skill_info: SkillInfo | None = None
+    skill_discovery: SkillDiscovery | None = None
     effective_system_prompt: str = ""
     mcp_prompts: list[MCPPrompt] = field(default_factory=list)
     prompt_name: str | None = None

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.1}/src/pytest_skill_engineering/core/serialization.py

@@ -56,7 +56,14 @@ def deserialize_suite_report(data: dict[str, Any]) -> SuiteReport:
     Reconstructs the full dataclass hierarchy from the serialized format.
     """
     from pytest_skill_engineering.copilot.requests import RequestAudit
-    from pytest_skill_engineering.core.result import EvalResult, ToolCall, Turn, UsageInfo
+    from pytest_skill_engineering.core.result import (
+        DiscoveredSkill,
+        EvalResult,
+        SkillDiscovery,
+        ToolCall,
+        Turn,
+        UsageInfo,
+    )
     from pytest_skill_engineering.reporting.collector import SuiteReport, TestReport
 
     # Reconstruct tests
@@ -214,6 +221,20 @@ def deserialize_suite_report(data: dict[str, Any]) -> SuiteReport:
                 )
 
             # Reconstruct agent result
+            discovery_data = _require_key(ar_data, "skill_discovery", context="EvalResult")
+            skill_discovery = None
+            if discovery_data is not None:
+                skill_discovery = SkillDiscovery(
+                    skills=[
+                        DiscoveredSkill(**skill)
+                        for skill in _require_key(
+                            discovery_data, "skills", context="SkillDiscovery"
+                        )
+                    ],
+                    warnings=_require_key(discovery_data, "warnings", context="SkillDiscovery"),
+                    errors=_require_key(discovery_data, "errors", context="SkillDiscovery"),
+                    complete=_require_key(discovery_data, "complete", context="SkillDiscovery"),
+                )
             eval_result = EvalResult(
                 turns=turns,
                 success=_require_key(ar_data, "success", context="EvalResult"),
@@ -228,6 +249,7 @@ def deserialize_suite_report(data: dict[str, Any]) -> SuiteReport:
                 assertions=assertions,
                 available_tools=available_tools,
                 skill_info=skill_info,
+                skill_discovery=skill_discovery,
                 effective_system_prompt=_require_key(
                     ar_data, "effective_system_prompt", context="EvalResult"
                 ),