pytest-skill-engineering 1.0.0__tar.gz → 1.0.2__tar.gz

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pytest-skill-engineering
-Version: 1.0.0
+Version: 1.0.2
 Summary: Reliable Copilot execution and evidence for coding-agent-led testing of MCP servers, CLI tools, skills, and custom agents.
 Project-URL: Homepage, https://github.com/sbroenne/pytest-skill-engineering
 Project-URL: Repository, https://github.com/sbroenne/pytest-skill-engineering

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pytest-skill-engineering"
-version = "1.0.0"
+version = "1.0.2"
 description = "Reliable Copilot execution and evidence for coding-agent-led testing of MCP servers, CLI tools, skills, and custom agents."
 readme = "README.md"
 license = { text = "MIT" }

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/copilot/__init__.py

@@ -22,11 +22,14 @@ from pytest_skill_engineering.copilot.personas import (
 from pytest_skill_engineering.copilot.requests import RequestAudit
 from pytest_skill_engineering.copilot.result import CopilotResult, UsageInfo
 from pytest_skill_engineering.core.evals import load_custom_agent, load_custom_agents
+from pytest_skill_engineering.core.result import DiscoveredSkill, SkillDiscovery
 
 __all__ = [
     "CopilotEval",
     "CopilotResult",
+    "DiscoveredSkill",
     "RequestAudit",
+    "SkillDiscovery",
     "UsageInfo",
     "ClaudeCodePersona",
     "CopilotCLIPersona",

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/copilot/eval.py

@@ -139,7 +139,7 @@ class CopilotEval:
     # display_name, tools, mcp_servers, infer)
     custom_agents: list[CopilotCustomAgentConfig] = field(default_factory=list)
 
-    # Skill directories
+    # Explicit skills are enabled and checked before model execution.
     skill_directories: list[str] = field(default_factory=list)
     disabled_skills: list[str] = field(default_factory=list)
 
@@ -235,8 +235,24 @@ class CopilotEval:
             config["hooks"] = self.hooks
 
         # Apply extra_config passthrough
+        if (
+            self.skill_directories
+            and "skill_directories" in self.extra_config
+            and self.extra_config["skill_directories"] != self.skill_directories
+        ):
+            raise ValueError("extra_config must not override explicitly supplied skill_directories")
         config.update(self.extra_config)
 
+        if config.get("skill_directories"):
+            if config.get("enable_skills") is False:
+                raise ValueError(
+                    "Explicit skill_directories conflict with enable_skills=False; "
+                    "remove the directories for a no-skill baseline, or enable skills"
+                )
+            if config.get("enable_skills") is not None and config["enable_skills"] is not True:
+                raise ValueError("enable_skills must be a boolean or None")
+            config["enable_skills"] = True
+
         if self.client_mode == "empty":
             isolated = {
                 "enable_config_discovery": False,

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/copilot/fixtures.py

@@ -116,6 +116,7 @@ def _convert_to_aitest(
         stop_reason=result.stop_reason,
         usage=list(result.usage),
         tool_calls_admitted=result.tool_calls_admitted,
+        skill_discovery=deepcopy(result.skill_discovery),
         configuration=deepcopy(
             {
                 "name": agent.name,

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/copilot/result.py

@@ -12,7 +12,13 @@ from typing import Any
 
 from pytest_skill_engineering.copilot.contracts import CopilotResultAgent, SubagentInvocation
 from pytest_skill_engineering.copilot.requests import RequestAudit
-from pytest_skill_engineering.core.result import StopReason, ToolCall, Turn, UsageInfo
+from pytest_skill_engineering.core.result import (
+    SkillDiscovery,
+    StopReason,
+    ToolCall,
+    Turn,
+    UsageInfo,
+)
 
 __all__ = [
     "CopilotResult",
@@ -68,6 +74,7 @@ class CopilotResult:
     request_audit: list[RequestAudit] = field(default_factory=list)
     stop_reason: StopReason | None = None
     tool_calls_admitted: int = 0
+    skill_discovery: SkillDiscovery | None = None
 
     @property
     def evidence_complete(self) -> bool:

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/copilot/runner.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import TYPE_CHECKING
 
@@ -27,6 +28,8 @@ from pytest_skill_engineering.copilot.controls import RunControls
 from pytest_skill_engineering.copilot.events import EventMapper
 from pytest_skill_engineering.copilot.requests import RequestAuditHandler
 from pytest_skill_engineering.copilot.result import CopilotResult, StopReason
+from pytest_skill_engineering.copilot.skills import discover_requested_skills, requested_skill_files
+from pytest_skill_engineering.core.result import SkillDiscovery
 
 if TYPE_CHECKING:
     from copilot.client import CopilotClient
@@ -80,8 +83,20 @@ async def run_copilot(agent: CopilotEvalConfig, prompt: str) -> CopilotResult:
     reason: StopReason = "completed"
     error: str | None = None
     cleanup_errors: list[str] = []
+    skill_discovery: SkillDiscovery | None = None
+    send_started = False
     try:
         session_config = agent.build_session_config()
+        skill_directories = session_config.get("skill_directories") or []
+        requested_skills: dict[Path, str] = {}
+        if skill_directories:
+            skill_discovery = SkillDiscovery()
+            if not isinstance(skill_directories, list):
+                raise ValueError("skill_directories must be a list of directory paths")
+            requested_skills = requested_skill_files(skill_directories)
+            session_config["skill_directories"] = [
+                str(Path(directory).resolve()) for directory in skill_directories
+            ]
         caller_hooks = dict(session_config.get("hooks") or {})
         session_config["hooks"] = dict(caller_hooks)
         # Empty mode must not read persona instruction files or inject tools.
@@ -123,12 +138,20 @@ async def run_copilot(agent: CopilotEvalConfig, prompt: str) -> CopilotResult:
         )
 
         async def execute() -> None:
-            nonlocal session
+            nonlocal session, send_started
             assert client is not None
             async with asyncio.timeout(min(60, agent.timeout_s)):
                 await client.start()
             async with asyncio.timeout(min(30, agent.timeout_s)):
                 session = await client.create_session(**session_config)
+                if skill_discovery is not None:
+                    await discover_requested_skills(
+                        session,
+                        requested_skills,
+                        session_config.get("disabled_skills") or [],
+                        skill_discovery,
+                    )
+            send_started = True
             await session.send_and_wait(prompt, timeout=agent.timeout_s)
 
         execution = asyncio.create_task(execute())
@@ -158,6 +181,9 @@ async def run_copilot(agent: CopilotEvalConfig, prompt: str) -> CopilotResult:
     except Exception as exc:
         logger.error("Copilot execution failed: %s", exc)
         reason, error = "execution_error", str(exc)
+        if skill_discovery is not None and not skill_discovery.complete:
+            error = f"Skill setup failed before model execution: {exc}"
+            skill_discovery.errors.append(error)
     finally:
         controls.closed = True
         for watcher in watchers:
@@ -210,12 +236,13 @@ async def run_copilot(agent: CopilotEvalConfig, prompt: str) -> CopilotResult:
 
     result = mapper.build()
     result.agent = agent
+    result.skill_discovery = skill_discovery
     result.tool_calls_admitted = controls.admitted
     if result.capture_errors:
         result.capture_errors.extend(controls.incomplete_diagnostics(abort_phase=reason))
     if audit is not None and capture_requests:
         result.request_audit = list(audit.records)
-        if audit.error or (session is not None and not audit.observed_requests):
+        if audit.error or (send_started and not audit.observed_requests):
             audit_error = audit.error or "No outbound model requests captured; audit unsupported"
             result.capture_errors.append(audit_error)
             if reason == "completed" or audit.error:

pytest_skill_engineering-1.0.2/src/pytest_skill_engineering/copilot/skills.py

@@ -0,0 +1,104 @@
+"""Validate explicit skill inputs and verify native SDK availability before sending."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pytest_skill_engineering.core.result import DiscoveredSkill, SkillDiscovery
+from pytest_skill_engineering.core.skill import Skill, SkillError
+
+if TYPE_CHECKING:
+    from copilot.session import CopilotSession
+
+logger = logging.getLogger(__name__)
+
+
+def requested_skill_files(directories: list[str]) -> dict[Path, str]:
+    """Validate individual skill directories or parents of immediate skill directories.
+
+    Relative directories are resolved from the Python caller's current directory,
+    then passed to the SDK as absolute paths.
+    """
+    requested: dict[Path, str] = {}
+    names: dict[str, Path] = {}
+    for raw_directory in directories:
+        if not isinstance(raw_directory, str) or not raw_directory.strip():
+            raise SkillError("skill_directories entries must be non-empty directory paths")
+        directory = Path(raw_directory).resolve()
+        if not directory.is_dir():
+            raise SkillError(
+                f"Requested skill directory does not exist or is not a directory: {directory}"
+            )
+        try:
+            candidates = (
+                [directory]
+                if (directory / "SKILL.md").exists()
+                else [
+                    child
+                    for child in sorted(directory.iterdir())
+                    if child.is_dir() and (child / "SKILL.md").exists()
+                ]
+            )
+            if not candidates:
+                raise SkillError(f"No SKILL.md found in requested skill directory: {directory}")
+            for candidate in candidates:
+                skill = Skill.from_path(candidate)
+                path = (skill.path / "SKILL.md").resolve()
+                if skill.name in names and names[skill.name] != path:
+                    raise SkillError(
+                        f"Duplicate requested skill name '{skill.name}': "
+                        f"{names[skill.name]} and {path}"
+                    )
+                names[skill.name] = path
+                requested[path] = skill.name
+        except OSError as exc:
+            raise SkillError(f"Cannot read requested skill directory {directory}: {exc}") from exc
+    return requested
+
+
+async def discover_requested_skills(
+    session: CopilotSession,
+    requested: dict[Path, str],
+    disabled: list[str],
+    evidence: SkillDiscovery,
+) -> None:
+    """Record SDK discovery and reject missing or unexpectedly disabled requested skills."""
+    await session.rpc.skills.ensure_loaded()
+    diagnostics = await session.rpc.skills.reload()
+    evidence.warnings = list(diagnostics.warnings)
+    evidence.errors = list(diagnostics.errors)
+    for warning in evidence.warnings:
+        logger.warning("SDK skill discovery: %s", warning)
+    listed = await session.rpc.skills.list()
+    evidence.skills = [
+        DiscoveredSkill(
+            name=skill.name,
+            description=skill.description,
+            source=skill.source.value,
+            enabled=skill.enabled,
+            user_invocable=skill.user_invocable,
+            path=skill.path,
+            plugin_name=skill.plugin_name,
+            command_name=skill.command_name,
+            argument_hint=skill.argument_hint,
+        )
+        for skill in listed.skills
+    ]
+    evidence.complete = True
+    available = {
+        (Path(skill.path).resolve(), skill.name): skill
+        for skill in evidence.skills
+        if skill.path is not None
+    }
+    for path, name in requested.items():
+        skill = available.get((path, name))
+        if skill is None:
+            evidence.errors.append(
+                f"Requested skill '{name}' was not discovered by the SDK: {path}"
+            )
+        elif not skill.enabled and name not in disabled:
+            evidence.errors.append(f"Requested skill '{name}' is unexpectedly disabled: {path}")
+    if evidence.errors:
+        raise SkillError("Skill setup failed before model execution: " + "; ".join(evidence.errors))

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/core/__init__.py

@@ -17,10 +17,12 @@ from pytest_skill_engineering.core.prompt import (
     load_system_prompts,
 )
 from pytest_skill_engineering.core.result import (
+    DiscoveredSkill,
     EvalResult,
     ImageContent,
     MCPPrompt,
     MCPPromptArgument,
+    SkillDiscovery,
     SkillInfo,
     SubagentInvocation,
     ToolCall,
@@ -38,6 +40,7 @@ from pytest_skill_engineering.core.skill_grading import export_grading
 __all__ = [
     "AITestError",
     "EvalResult",
+    "DiscoveredSkill",
     "EngineTimeoutError",
     "HookDefinition",
     "ImageContent",
@@ -48,6 +51,7 @@ __all__ = [
     "Prompt",
     "ServerStartError",
     "Skill",
+    "SkillDiscovery",
     "SkillError",
     "SkillInfo",
     "SkillMetadata",

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/core/plugin.py

@@ -341,7 +341,7 @@ def _discover_skills(plugin_dir: Path) -> list[Skill]:
 
 
 def _validate_skill_reference_names(skills: list[Skill]) -> None:
-    """Reject duplicate reference basenames across loaded skills."""
+    """Reject duplicate relative reference paths across loaded skills."""
     owners: dict[str, str] = {}
     for skill in skills:
         for reference_name in skill.references:

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/core/result.py

@@ -181,6 +181,31 @@ class SkillInfo:
         return f"SkillInfo({self.name}{refs})"
 
 
+@dataclass(slots=True, frozen=True)
+class DiscoveredSkill:
+    """Skill metadata actually returned by the public SDK, not observed use."""
+
+    name: str
+    description: str
+    source: str
+    enabled: bool
+    user_invocable: bool
+    path: str | None = None
+    plugin_name: str | None = None
+    command_name: str | None = None
+    argument_hint: str | None = None
+
+
+@dataclass(slots=True)
+class SkillDiscovery:
+    """Pre-execution discovery evidence, including incomplete or failed checks."""
+
+    skills: list[DiscoveredSkill] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+    complete: bool = False
+
+
 @dataclass(slots=True)
 class ClarificationStats:
     """Statistics about clarification requests detected during execution.
@@ -280,6 +305,7 @@ class EvalResult:
     # Recorded interface metadata.
     available_tools: list[ToolInfo] = field(default_factory=list)
     skill_info: SkillInfo | None = None
+    skill_discovery: SkillDiscovery | None = None
     effective_system_prompt: str = ""
     mcp_prompts: list[MCPPrompt] = field(default_factory=list)
     prompt_name: str | None = None

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/core/serialization.py

@@ -56,7 +56,14 @@ def deserialize_suite_report(data: dict[str, Any]) -> SuiteReport:
     Reconstructs the full dataclass hierarchy from the serialized format.
     """
     from pytest_skill_engineering.copilot.requests import RequestAudit
-    from pytest_skill_engineering.core.result import EvalResult, ToolCall, Turn, UsageInfo
+    from pytest_skill_engineering.core.result import (
+        DiscoveredSkill,
+        EvalResult,
+        SkillDiscovery,
+        ToolCall,
+        Turn,
+        UsageInfo,
+    )
     from pytest_skill_engineering.reporting.collector import SuiteReport, TestReport
 
     # Reconstruct tests
@@ -214,6 +221,20 @@ def deserialize_suite_report(data: dict[str, Any]) -> SuiteReport:
                 )
 
             # Reconstruct agent result
+            discovery_data = _require_key(ar_data, "skill_discovery", context="EvalResult")
+            skill_discovery = None
+            if discovery_data is not None:
+                skill_discovery = SkillDiscovery(
+                    skills=[
+                        DiscoveredSkill(**skill)
+                        for skill in _require_key(
+                            discovery_data, "skills", context="SkillDiscovery"
+                        )
+                    ],
+                    warnings=_require_key(discovery_data, "warnings", context="SkillDiscovery"),
+                    errors=_require_key(discovery_data, "errors", context="SkillDiscovery"),
+                    complete=_require_key(discovery_data, "complete", context="SkillDiscovery"),
+                )
             eval_result = EvalResult(
                 turns=turns,
                 success=_require_key(ar_data, "success", context="EvalResult"),
@@ -228,6 +249,7 @@ def deserialize_suite_report(data: dict[str, Any]) -> SuiteReport:
                 assertions=assertions,
                 available_tools=available_tools,
                 skill_info=skill_info,
+                skill_discovery=skill_discovery,
                 effective_system_prompt=_require_key(
                     ar_data, "effective_system_prompt", context="EvalResult"
                 ),

{pytest_skill_engineering-1.0.0 → pytest_skill_engineering-1.0.2}/src/pytest_skill_engineering/core/skill.py

@@ -28,6 +28,7 @@ Example SKILL.md:
 from __future__ import annotations
 
 import re
+import stat
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -163,7 +164,8 @@ class Skill:
             path: Path to skill directory or SKILL.md file
 
         Returns:
-            Loaded Skill instance
+            Loaded Skill instance. Reference keys are slash-separated paths
+            relative to references/, including any nested subdirectories.
 
         Raises:
             SkillError: If skill cannot be loaded or is invalid
@@ -197,7 +199,7 @@ class Skill:
 
         for name in ("references", "scripts", "assets"):
             component = skill_dir / name
-            if component.exists() and not component.is_dir():
+            if (component.exists() or component.is_symlink()) and not component.is_dir():
                 raise SkillError(f"{component}: skill component must be a directory")
 
         # Load references if directory exists
@@ -320,27 +322,59 @@ def _parse_skill_md(content: str) -> tuple[SkillMetadata, str]:
 
 
 def _load_references(refs_dir: Path) -> dict[str, str]:
-    """Load all files from references/ directory.
+    """Load Markdown files recursively, without escaping references/ or following cycles.
 
     Returns:
-        Dict mapping filename to content
+        Dict mapping slash-separated paths relative to references/ to content
     """
+    root = _resolve_reference_path(refs_dir, refs_dir.parent.resolve())
     references: dict[str, str] = {}
-
-    for file_path in refs_dir.iterdir():
-        if not file_path.is_file():
-            raise SkillError(f"Invalid references entry (must be a file): {file_path}")
-        if file_path.suffix.lower() != ".md":
-            raise SkillError(f"Invalid reference file '{file_path}': only .md files are allowed")
+    pending = [(refs_dir, frozenset({root}))]
+    while pending:
+        directory, ancestors = pending.pop()
         try:
-            content = file_path.read_text(encoding="utf-8")
-        except UnicodeDecodeError as exc:
-            raise SkillError(f"Reference file must be valid UTF-8 text: {file_path}") from exc
-        if not content.strip():
-            raise SkillError(f"Reference file must not be empty: {file_path}")
-        references[file_path.name] = content
+            entries = sorted(directory.iterdir())
+        except OSError as exc:
+            raise SkillError(f"Cannot read references directory {directory}: {exc}") from exc
+        for file_path in entries:
+            resolved = _resolve_reference_path(file_path, root)
+            try:
+                mode = resolved.stat().st_mode
+            except OSError as exc:
+                raise SkillError(f"Cannot inspect reference entry {file_path}: {exc}") from exc
+            if stat.S_ISDIR(mode):
+                if resolved in ancestors:
+                    raise SkillError(f"Reference directory cycle: {file_path}")
+                pending.append((file_path, ancestors | {resolved}))
+                continue
+            if not stat.S_ISREG(mode):
+                raise SkillError(f"Invalid references entry (must be a file): {file_path}")
+            if file_path.suffix.lower() != ".md":
+                raise SkillError(
+                    f"Invalid reference file '{file_path}': only .md files are allowed"
+                )
+            try:
+                content = resolved.read_text(encoding="utf-8")
+            except UnicodeDecodeError as exc:
+                raise SkillError(f"Reference file must be valid UTF-8 text: {file_path}") from exc
+            except OSError as exc:
+                raise SkillError(f"Cannot read reference file {file_path}: {exc}") from exc
+            if not content.strip():
+                raise SkillError(f"Reference file must not be empty: {file_path}")
+            references[file_path.relative_to(refs_dir).as_posix()] = content
+
+    return dict(sorted(references.items()))
 
-    return references
+
+def _resolve_reference_path(path: Path, root: Path) -> Path:
+    """Resolve links before traversal or reads, keeping their targets within the root."""
+    try:
+        resolved = path.resolve(strict=True)
+    except (OSError, RuntimeError) as exc:
+        raise SkillError(f"Cannot resolve reference path {path}: {exc}") from exc
+    if not resolved.is_relative_to(root):
+        raise SkillError(f"Reference path escapes its root {root}: {path}")
+    return resolved
 
 
 _SCRIPT_EXTENSIONS = frozenset({".py", ".sh", ".js", ".bash"})
@@ -375,7 +409,8 @@ def _load_scripts(scripts_dir: Path) -> dict[str, str]:
 def load_skill(path: Path | str) -> Skill:
     """Load a skill from a path.
 
-    Convenience function wrapping Skill.from_path().
+    Convenience function wrapping Skill.from_path(). This is the runner's local
+    skill validation, without SDK startup or model execution.
 
     Args:
         path: Path to skill directory or SKILL.md file