pytest-skill-engineering 0.6.22__tar.gz → 1.0.0__tar.gz

{pytest_skill_engineering-0.6.22 → pytest_skill_engineering-1.0.0}/.gitignore

@@ -1,67 +1,65 @@
-# Python
-__pycache__/
-*.py[cod]
-*$py.class
-*.so
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# Virtual environments
-.venv/
-venv/
-ENV/
-env/
-
-# IDE
-.idea/
-.vscode/
-*.swp
-*.swo
-*~
-
-# Testing
-.pytest_cache/
-.coverage
-htmlcov/
-.tox/
-.nox/
-
-# Type checking
-.mypy_cache/
-
-# Test outputs
-test_results/
-aitest-reports/
-tests/fixtures/reports/html/
-
-# Node.js (Tailwind build)
-node_modules/
-src/pytest_aitest/templates/node_modules/
-package-lock.json
-
-# Environment
-.env
-.env.local
-
-# OS
-.DS_Store
-Thumbs.db
-site/
-.cache/
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+
+# Type checking
+.mypy_cache/
+
+# Test outputs
+test_results/
+aitest-reports/
+
+# Node.js
+node_modules/
+package-lock.json
+
+# Environment
+.env
+.env.local
+
+# OS
+.DS_Store
+Thumbs.db
+site/
+.cache/
 # Squad: ignore runtime state (logs, inbox, sessions)
 .squad/orchestration-log/
 .squad/log/
@@ -71,4 +69,4 @@ site/
 .squad-workstream
 
 # pytest-skill-engineering test scratch (some server tests create a cwd dir)
-.pytest-execution-servers/
+.pytest-execution-servers/

pytest_skill_engineering-1.0.0/PKG-INFO

@@ -0,0 +1,182 @@
+Metadata-Version: 2.5
+Name: pytest-skill-engineering
+Version: 1.0.0
+Summary: Reliable Copilot execution and evidence for coding-agent-led testing of MCP servers, CLI tools, skills, and custom agents.
+Project-URL: Homepage, https://github.com/sbroenne/pytest-skill-engineering
+Project-URL: Repository, https://github.com/sbroenne/pytest-skill-engineering
+Author: Stefan Brunner
+License: MIT
+License-File: LICENSE
+Keywords: agents,ai,custom-agents,llm,mcp,pytest,skill-engineering,skills,testing
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: github-copilot-sdk>=1.0.16
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: mcp>=2.2.0
+Requires-Dist: pydantic>=2.13.5
+Requires-Dist: pytest-asyncio>=1.4.0
+Requires-Dist: pytest>=9.1.1
+Requires-Dist: python-frontmatter>=1.3.0
+Requires-Dist: tomlkit>=0.15.1
+Requires-Dist: trio>=0.34.0
+Requires-Dist: websockets>=17.1
+Provides-Extra: dev
+Requires-Dist: pre-commit>=4.6.2; extra == 'dev'
+Requires-Dist: pyright>=1.1.414; extra == 'dev'
+Requires-Dist: pytest-cov>=7.1.0; extra == 'dev'
+Requires-Dist: python-dotenv>=1.2.3; extra == 'dev'
+Requires-Dist: ruff>=0.16.9; extra == 'dev'
+Requires-Dist: typeguard>=4.6.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: cairosvg>=2.9.1; extra == 'docs'
+Requires-Dist: mkdocs-material<9.8,>=9.7.7; extra == 'docs'
+Requires-Dist: mkdocs>=1.6.1; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=1.0.6; extra == 'docs'
+Requires-Dist: pillow>=12.3.0; extra == 'docs'
+Provides-Extra: test
+Requires-Dist: syrupy<7,>=6.1.1; extra == 'test'
+Description-Content-Type: text/markdown
+
+# pytest-skill-engineering
+
+[![PyPI version](https://img.shields.io/pypi/v/pytest-skill-engineering)](https://pypi.org/project/pytest-skill-engineering/)
+[![CI](https://github.com/sbroenne/pytest-skill-engineering/actions/workflows/ci.yml/badge.svg)](https://github.com/sbroenne/pytest-skill-engineering/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+**A test runner for people working with coding agents.**
+
+Test whether real GitHub Copilot sessions can use your MCP servers, CLI tools,
+skills, system prompts, and custom agents. Write a user task, execute it through
+pytest, and verify the actual output with ordinary assertions.
+
+The framework runs the task and records what happened. Your existing coding agent
+investigates the evidence, reads the source, and helps you improve the interface.
+**Version 1.0 removes AI judging, report dashboards, rankings, and skill refinement.**
+See the [migration guide](docs/migration.md) for this breaking change.
+
+## Why this exists
+
+Your tool can pass its own tests while Copilot chooses the wrong operation,
+supplies the wrong arguments, or misunderstands its descriptions. That is a
+different question from whether the tool implementation works.
+
+Use real task execution to test the AI-facing interface, and independent output
+checks to establish correctness. A completed session or a tool call by itself
+does not prove that the requested task succeeded.
+
+## Quick start
+
+You need Python 3.11+, [uv](https://docs.astral.sh/uv/), and a GitHub account with
+Copilot access.
+
+```powershell
+uv add pytest-skill-engineering
+gh auth login --hostname github.com
+uv run pytest-skill-engineering init
+uv run pytest-skill-engineering doctor
+uv run python -m pytest "tests\test_copilot_eval.py" -v
+```
+
+`init` creates a Todo MCP starter test, explicit JSON evidence settings, and
+explicit starter-model rates in `pricing.toml`. It refuses conflicting settings
+or an existing starter. Live execution can consume Copilot premium requests.
+
+pytest shows ordinary test results and assertion failures. Give your coding
+agent `aitest-reports\results.json` and the relevant source to investigate what
+happened. No dashboard or second AI judgment is generated.
+
+## What we learned by testing our own idea
+
+We evaluated a companion skill for coding agents using this framework and chose
+not to ship it. The comparison did not demonstrate a benefit; adding general
+test-writing advice was not evidence that users needed another product layer.
+Use the documentation, examples, and native evidence directly.
+
+The [case study](docs/use-cases/companion-skill.md) records the actual results,
+failures, limitations, and removal decision. Its
+[historical experiment](examples/skill-dogfood/) remains reproducible with a
+frozen test fixture, not an installable companion skill. Default sample checks
+are offline; live execution is explicit. The [quickstart](examples/quickstart/)
+is the smaller first example.
+
+Testing your own domain skills remains a supported framework capability.
+
+## A test that checks the result
+
+```python
+from __future__ import annotations
+
+import subprocess
+import sys
+
+from pytest_skill_engineering.copilot import CopilotEval
+
+
+async def test_addition(copilot_eval, tmp_path):
+    agent = CopilotEval(
+        name="addition",
+        model="gpt-5.6-luna",
+        instructions="Write Python code and save the requested file.",
+        working_directory=str(tmp_path),
+    )
+    result = await copilot_eval(agent, "Create calc.py with add(a, b) returning a + b.")
+    assert result.success, result.error
+    assert (tmp_path / "calc.py").is_file()
+    checked = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "from calc import add; assert add(2, 3) == 5; assert add(-2, 2) == 0",
+        ],
+        cwd=tmp_path,
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+    assert checked.returncode == 0, checked.stderr
+```
+
+Use your approved isolation when executing generated code. A temporary directory
+is not a security sandbox.
+
+## What stays in the framework
+
+| Capability | What it provides |
+| --- | --- |
+| `CopilotEval` and `copilot_eval` | Real SDK sessions with explicit configuration |
+| MCP, CLI, skills, plugins, custom agents | The interfaces and definitions under test |
+| Execution controls | Time, usage, request, tool, and permission controls; one attempt per execution |
+| Captured evidence | Configuration, calls, arguments, outputs, completion flags, errors, and usage |
+| Ordinary pytest checks | Consumer-owned verification and recorded properties |
+| `ab_run` and repetitions | Isolated working directories and repeated observations |
+| Structured JSON | Captured execution and ordinary pytest outcomes, without rankings or advice |
+| Pricing | Explicit USD estimates and recorded Copilot premium requests |
+
+A/B entries share one pytest outcome. Record side-specific checks; do not read
+that shared outcome as independent per-side success or causal improvement.
+Missing prices and incomplete evidence are recorded explicitly.
+
+## Let your coding agent investigate
+
+Ask your existing coding agent to inspect a failed test, its saved JSON evidence,
+and the source. It can explain the supported cause and help fix it, without
+changing the test's success criteria after seeing the failure.
+
+Reading current schema-4.0 evidence needs no authentication or paid call.
+The framework records results; the coding agent interprets them. Subjective
+review is advice, not an automatic pytest verdict.
+
+Read the [full documentation](https://sbroenne.github.io/pytest-skill-engineering/).
+
+## License
+
+MIT. Inspired by [agent-benchmark](https://github.com/mykhaliev/agent-benchmark).

pytest_skill_engineering-1.0.0/README.md

@@ -0,0 +1,134 @@
+# pytest-skill-engineering
+
+[![PyPI version](https://img.shields.io/pypi/v/pytest-skill-engineering)](https://pypi.org/project/pytest-skill-engineering/)
+[![CI](https://github.com/sbroenne/pytest-skill-engineering/actions/workflows/ci.yml/badge.svg)](https://github.com/sbroenne/pytest-skill-engineering/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+**A test runner for people working with coding agents.**
+
+Test whether real GitHub Copilot sessions can use your MCP servers, CLI tools,
+skills, system prompts, and custom agents. Write a user task, execute it through
+pytest, and verify the actual output with ordinary assertions.
+
+The framework runs the task and records what happened. Your existing coding agent
+investigates the evidence, reads the source, and helps you improve the interface.
+**Version 1.0 removes AI judging, report dashboards, rankings, and skill refinement.**
+See the [migration guide](docs/migration.md) for this breaking change.
+
+## Why this exists
+
+Your tool can pass its own tests while Copilot chooses the wrong operation,
+supplies the wrong arguments, or misunderstands its descriptions. That is a
+different question from whether the tool implementation works.
+
+Use real task execution to test the AI-facing interface, and independent output
+checks to establish correctness. A completed session or a tool call by itself
+does not prove that the requested task succeeded.
+
+## Quick start
+
+You need Python 3.11+, [uv](https://docs.astral.sh/uv/), and a GitHub account with
+Copilot access.
+
+```powershell
+uv add pytest-skill-engineering
+gh auth login --hostname github.com
+uv run pytest-skill-engineering init
+uv run pytest-skill-engineering doctor
+uv run python -m pytest "tests\test_copilot_eval.py" -v
+```
+
+`init` creates a Todo MCP starter test, explicit JSON evidence settings, and
+explicit starter-model rates in `pricing.toml`. It refuses conflicting settings
+or an existing starter. Live execution can consume Copilot premium requests.
+
+pytest shows ordinary test results and assertion failures. Give your coding
+agent `aitest-reports\results.json` and the relevant source to investigate what
+happened. No dashboard or second AI judgment is generated.
+
+## What we learned by testing our own idea
+
+We evaluated a companion skill for coding agents using this framework and chose
+not to ship it. The comparison did not demonstrate a benefit; adding general
+test-writing advice was not evidence that users needed another product layer.
+Use the documentation, examples, and native evidence directly.
+
+The [case study](docs/use-cases/companion-skill.md) records the actual results,
+failures, limitations, and removal decision. Its
+[historical experiment](examples/skill-dogfood/) remains reproducible with a
+frozen test fixture, not an installable companion skill. Default sample checks
+are offline; live execution is explicit. The [quickstart](examples/quickstart/)
+is the smaller first example.
+
+Testing your own domain skills remains a supported framework capability.
+
+## A test that checks the result
+
+```python
+from __future__ import annotations
+
+import subprocess
+import sys
+
+from pytest_skill_engineering.copilot import CopilotEval
+
+
+async def test_addition(copilot_eval, tmp_path):
+    agent = CopilotEval(
+        name="addition",
+        model="gpt-5.6-luna",
+        instructions="Write Python code and save the requested file.",
+        working_directory=str(tmp_path),
+    )
+    result = await copilot_eval(agent, "Create calc.py with add(a, b) returning a + b.")
+    assert result.success, result.error
+    assert (tmp_path / "calc.py").is_file()
+    checked = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "from calc import add; assert add(2, 3) == 5; assert add(-2, 2) == 0",
+        ],
+        cwd=tmp_path,
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+    assert checked.returncode == 0, checked.stderr
+```
+
+Use your approved isolation when executing generated code. A temporary directory
+is not a security sandbox.
+
+## What stays in the framework
+
+| Capability | What it provides |
+| --- | --- |
+| `CopilotEval` and `copilot_eval` | Real SDK sessions with explicit configuration |
+| MCP, CLI, skills, plugins, custom agents | The interfaces and definitions under test |
+| Execution controls | Time, usage, request, tool, and permission controls; one attempt per execution |
+| Captured evidence | Configuration, calls, arguments, outputs, completion flags, errors, and usage |
+| Ordinary pytest checks | Consumer-owned verification and recorded properties |
+| `ab_run` and repetitions | Isolated working directories and repeated observations |
+| Structured JSON | Captured execution and ordinary pytest outcomes, without rankings or advice |
+| Pricing | Explicit USD estimates and recorded Copilot premium requests |
+
+A/B entries share one pytest outcome. Record side-specific checks; do not read
+that shared outcome as independent per-side success or causal improvement.
+Missing prices and incomplete evidence are recorded explicitly.
+
+## Let your coding agent investigate
+
+Ask your existing coding agent to inspect a failed test, its saved JSON evidence,
+and the source. It can explain the supported cause and help fix it, without
+changing the test's success criteria after seeing the failure.
+
+Reading current schema-4.0 evidence needs no authentication or paid call.
+The framework records results; the coding agent interprets them. Subjective
+review is advice, not an automatic pytest verdict.
+
+Read the [full documentation](https://sbroenne.github.io/pytest-skill-engineering/).
+
+## License
+
+MIT. Inspired by [agent-benchmark](https://github.com/mykhaliev/agent-benchmark).

{pytest_skill_engineering-0.6.22 → pytest_skill_engineering-1.0.0}/pyproject.toml

@@ -1,11 +1,11 @@
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling>=1.32.4"]
 build-backend = "hatchling.build"
 
 [project]
 name = "pytest-skill-engineering"
-version = "0.6.22"
-description = "The testing framework for skill engineering. Test tool descriptions, prompt templates, agent skills, and custom agents with real LLMs. AI analyzes results and tells you what to fix."
+version = "1.0.0"
+description = "Reliable Copilot execution and evidence for coding-agent-led testing of MCP servers, CLI tools, skills, and custom agents."
 readme = "README.md"
 license = { text = "MIT" }
 requires-python = ">=3.11"
@@ -26,45 +26,40 @@ classifiers = [
     "Topic :: Software Development :: Testing",
 ]
 dependencies = [
-    "pytest>=9.0",
-    "mcp>=2.1.1",
-    "pydantic>=2.0",
-    "markdown>=3.10.2",
-    "htpy>=25.12.0",
-    "mdutils>=1.8.1",
-    "python-frontmatter>=1.1.0",
-    "nh3>=0.3.3",
-    "github-copilot-sdk>=1.0.13",
+    "pytest>=9.1.1",
+    "mcp>=2.2.0",
+    "pydantic>=2.13.5",
+    "python-frontmatter>=1.3.0",
+    "github-copilot-sdk>=1.0.16",
     "httpx>=0.28.1",
-    "trio>=0.33.0",
+    "trio>=0.34.0",
     "tomlkit>=0.15.1",
-    "pytest-asyncio>=1.3",
-    "websockets>=15.0.1",
+    "pytest-asyncio>=1.4.0",
+    "websockets>=17.1",
 ]
 
 [project.optional-dependencies]
 dev = [
-    "pytest-cov>=7.0",
-    "python-dotenv>=1.2",
-    "ruff>=0.16.6",
-    "pyright>=1.1.408",
-    "pre-commit>=4.5",
-    "typeguard>=4.5",
+    "pytest-cov>=7.1.0",
+    "python-dotenv>=1.2.3",
+    "ruff>=0.16.9",
+    "pyright>=1.1.414",
+    "pre-commit>=4.6.2",
+    "typeguard>=4.6.0",
 ]
 test = [
-    "syrupy>=6.0.0,<7",
+    "syrupy>=6.1.1,<7",
 ]
 docs = [
-    "mkdocs>=1.6",
-    "mkdocs-material>=9.6,<9.8",
-    "mkdocstrings[python]>=0.24",
-    "pillow>=11.0",
-    "cairosvg>=2.7",
+    "mkdocs>=1.6.1",
+    "mkdocs-material>=9.7.7,<9.8",
+    "mkdocstrings[python]>=1.0.6",
+    "pillow>=12.3.0",
+    "cairosvg>=2.9.1",
 ]
 
 [project.scripts]
 pytest-skill-engineering = "pytest_skill_engineering.onboarding:main"
-pytest-skill-engineering-report = "pytest_skill_engineering.cli:main"
 
 [project.entry-points.pytest11]
 aitest = "pytest_skill_engineering.plugin"
@@ -83,9 +78,6 @@ packages = ["src/pytest_skill_engineering"]
 include = [
     "src/pytest_skill_engineering/**/*.py",
     "src/pytest_skill_engineering/**/*.md",
-    "src/pytest_skill_engineering/**/*.html",
-    "src/pytest_skill_engineering/**/*.css",
-    "src/pytest_skill_engineering/**/*.js",
 ]
 
 [tool.ruff]
@@ -106,7 +98,7 @@ quote-style = "double"
 indent-style = "space"
 
 [tool.pyright]
-include = ["src", "tests"]
+include = ["src", "tests", "examples/skill-dogfood"]
 pythonVersion = "3.11"
 typeCheckingMode = "basic"
 
@@ -125,8 +117,6 @@ filterwarnings = [
 # LLM auth is handled by the GitHub Copilot SDK (gh auth login or GITHUB_TOKEN)
 addopts = """
 --strict-markers
---aitest-summary-model=copilot/gpt-5.6-sol
---aitest-html=aitest-reports/report.html
 """
 markers = [
     "integration: marks tests as integration tests (require LLM credentials)",
@@ -138,7 +128,6 @@ markers = [
     "skill: tests with skills",
     "session_test: multi-turn session tests",
     "clarification: clarification detection tests",
-    "scoring: LLM scoring / rubric tests",
     "iterations: iteration reliability tests",
     "abtest: A/B server comparison tests",
     "cli: CLI server tests",
@@ -150,4 +139,8 @@ markers = [
 [dependency-groups]
 dev = [
     "pytest-skill-engineering[dev,test,docs]",
+    "pytest-skill-engineering-dogfood",
 ]
+
+[tool.uv.sources]
+pytest-skill-engineering-dogfood = { path = "examples/skill-dogfood", editable = true }

{pytest_skill_engineering-0.6.22 → pytest_skill_engineering-1.0.0}/src/pytest_skill_engineering/__init__.py

@@ -1,4 +1,4 @@
-"""pytest-skill-engineering: Pytest plugin for testing AI agents with MCP and CLI servers."""
+"""Copilot task execution and evidence for testing AI-facing tools and skills."""
 
 import logging
 
@@ -22,8 +22,6 @@ from pytest_skill_engineering.copilot import (  # noqa: E402
 )
 from pytest_skill_engineering.core import (  # noqa: E402
     AITestError,
-    BenchmarkComparison,
-    CaseBenchmark,
     EngineTimeoutError,
     EvalResult,
     HookDefinition,
@@ -33,25 +31,16 @@ from pytest_skill_engineering.core import (  # noqa: E402
     Plugin,
     PluginMetadata,
     Prompt,
-    RefinementResult,
-    RefinementSuggestion,
-    ScoreResult,
-    ScoringDimension,
     ServerStartError,
     Skill,
-    SkillBenchmarkResult,
-    SkillCaseResult,
     SkillError,
     SkillEvalCase,
-    SkillGradingResult,
     SkillInfo,
     SkillMetadata,
     SubagentInvocation,
     ToolCall,
     ToolInfo,
     Turn,
-    analyze_skill_failures,
-    assert_score,
     export_grading,
     has_skill_evals,
     load_custom_agent,
@@ -74,27 +63,18 @@ from pytest_skill_engineering.execution import (  # noqa: E402
     WaitStrategy,
 )
 
-# Hooks (for plugin extensibility)
-from pytest_skill_engineering.hooks import AitestHookSpec  # noqa: E402
-from pytest_skill_engineering.plugin import (  # noqa: E402
-    get_analysis_prompt,
-    get_analysis_prompt_details,
-)
-
 # Reporting
 from pytest_skill_engineering.reporting import (  # noqa: E402
     SuiteReport,
     TestReport,
     build_suite_report,
-    generate_html,
     generate_json,
+    load_suite_report,
 )
 
 __all__ = [  # noqa: RUF022
     # Core
     "AITestError",
-    "BenchmarkComparison",
-    "CaseBenchmark",
     "EngineTimeoutError",
     "EvalResult",
     "HookDefinition",
@@ -104,22 +84,16 @@ __all__ = [  # noqa: RUF022
     "Plugin",
     "PluginMetadata",
     "Prompt",
-    "RefinementResult",
-    "RefinementSuggestion",
     "ServerStartError",
     "Skill",
-    "SkillBenchmarkResult",
-    "SkillCaseResult",
     "SkillError",
     "SkillEvalCase",
-    "SkillGradingResult",
     "SkillInfo",
     "SkillMetadata",
     "SubagentInvocation",
     "ToolCall",
     "ToolInfo",
     "Turn",
-    "analyze_skill_failures",
     "export_grading",
     "has_skill_evals",
     "load_custom_agent",
@@ -152,16 +126,8 @@ __all__ = [  # noqa: RUF022
     "SuiteReport",
     "TestReport",
     "build_suite_report",
-    "generate_html",
     "generate_json",
-    # Hooks
-    "AitestHookSpec",
-    "get_analysis_prompt",
-    "get_analysis_prompt_details",
-    # Scoring
-    "ScoreResult",
-    "ScoringDimension",
-    "assert_score",
+    "load_suite_report",
 ]
 
 from importlib.metadata import version as _get_version  # noqa: E402