skill-evals 0.1.0__tar.gz

skill_evals-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Max van den Hoven
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

skill_evals-0.1.0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: skill-evals
+Version: 0.1.0
+Summary: Evaluate how coding-agent harnesses load and follow skills, in Docker sandboxes, traced with MLflow.
+Author: Max van den Hoven
+Author-email: Max van den Hoven <max.hoven@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Dist: jsonschema>=4.26.0
+Requires-Dist: mlflow>=3.17
+Requires-Dist: pandas>=3.0.6
+Requires-Dist: pyyaml>=6
+Requires-Python: >=3.12

skill_evals-0.1.0/pyproject.toml

@@ -0,0 +1,116 @@
+[project]
+name = "skill-evals"
+version = "0.1.0"
+description = "Evaluate how coding-agent harnesses load and follow skills, in Docker sandboxes, traced with MLflow."
+license = "MIT"
+license-files = ["LICENSE"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Testing",
+]
+requires-python = ">=3.12"
+dependencies = [
+    "jsonschema>=4.26.0",
+    "mlflow>=3.17",
+    "pandas>=3.0.6",
+    "pyyaml>=6",
+]
+
+[[project.authors]]
+name = "Max van den Hoven"
+email = "max.hoven@gmail.com"
+
+[project.scripts]
+skill-evals = "skill_evals.cli:main"
+
+[build-system]
+requires = ["uv_build>=0.11.2,<0.12.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "commitizen>=4.19.2",
+    "pre-commit>=4.6.2",
+    "pytest>=9.1.1",
+    "pytest-cov>=7.1.0",
+    "ruff>=0.17.0",
+    "ty>=0.0.86",
+    "types-jsonschema>=4.26.0.20261006",
+    "yamllint>=1.38.0",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-m 'not integration'"
+strict_markers = true
+markers = ["integration: needs sbx and Docker login, no model credentials; not run by default"]
+
+[tool.ty.environment]
+python-version = "3.12"
+
+[tool.ty.src]
+include = [
+    "src/**",
+    "example/scorers/**",
+    "tests/**",
+]
+exclude = ["example/fixtures/**"]
+
+[tool.ruff]
+line-length = 90
+indent-width = 4
+force-exclude = true
+extend-exclude = [
+    ".runs",
+    ".mlflow",
+    "example/fixtures",
+]
+
+[tool.ruff.lint]
+preview = true
+explicit-preview-rules = true
+select = [
+    "B",
+    "D",
+    "E",
+    "F",
+    "I",
+    "EM",
+    "T20",
+    "RET",
+    "ANN",
+    "DOC201",
+    "DOC202",
+    "DOC402",
+    "DOC501",
+    "DOC502",
+]
+ignore = [
+    "D100",
+    "D104",
+    "D203",
+    "D213",
+    "D400",
+    "D413",
+    "ANN002",
+    "ANN003",
+    "ANN401",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"src/skill_evals/{cli,run,score,init,mlflow_server}.py" = ["T201"]
+"tests/record_trace.py" = ["T201"]
+"src/skill_evals/harnesses/codex/tracing/files/home/.mlflow-codex/setup_tracing.py" = ["T201"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.commitizen]
+name = "cz_conventional_commits"
+version_provider = "uv"
+tag_format = "v$version"

skill_evals-0.1.0/pyproject.toml.orig

@@ -0,0 +1,132 @@
+# The `skill-evals` command (src/skill_evals/). It runs on the host; the only Python
+# that runs inside sandboxes is the Codex tracing helper of the shipped harnesses
+# (harnesses/codex/tracing/files/home/.mlflow-codex/setup_tracing.py). The example
+# project in example/ isn't part of the package.
+[project]
+name = "skill-evals"
+version = "0.1.0"
+description = "Evaluate how coding-agent harnesses load and follow skills, in Docker sandboxes, traced with MLflow."
+authors = [
+    { name = "Max van den Hoven", email = "max.hoven@gmail.com" },
+]
+license = "MIT"
+license-files = ["LICENSE"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Testing",
+]
+requires-python = ">=3.12"
+dependencies = [
+    "jsonschema>=4.26.0",  # Validates project files against the packaged schemas
+    "mlflow>=3.17",  # Tracking server, trace search and mlflow.genai.evaluate
+    "pandas>=3.0.6",  # Trace tables from mlflow.search_traces
+    "pyyaml>=6",  # Reads project files
+]
+
+[project.scripts]
+skill-evals = "skill_evals.cli:main"
+
+[build-system]
+requires = ["uv_build>=0.11.2,<0.12.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "commitizen>=4.19.2",
+    "pre-commit>=4.6.2",
+    "pytest>=9.1.1",
+    "pytest-cov>=7.1.0",
+    "ruff>=0.17.0",
+    "ty>=0.0.86",
+    "types-jsonschema>=4.26.0.20261006",
+    "yamllint>=1.38.0",
+]
+
+# Unit tests: no sbx, Docker, network, model credentials or MLflow server. Run them
+# with `make test`, which adds a coverage report.
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-m 'not integration'"
+strict_markers = true
+markers = [
+    "integration: needs sbx and Docker login, no model credentials; not run by default",
+]
+
+[tool.ty.environment]
+python-version = "3.12"  # The oldest supported version
+
+[tool.ty.src]
+include = [
+    "src/**",
+    "example/scorers/**",
+    "tests/**",
+]
+# Fixture files are test material for the agents (e.g. deliberately without docstrings).
+exclude = [
+    "example/fixtures/**",
+]
+
+[tool.ruff]
+line-length = 90
+indent-width = 4
+force-exclude = true  # Also skip excluded files that pre-commit passes explicitly
+extend-exclude = [
+    ".runs",
+    ".mlflow",
+    "example/fixtures",  # Test material for the agents, not our code
+]
+
+[tool.ruff.lint]
+# The DOC (pydoclint) rules are preview-only; enable exactly those, nothing else in preview.
+preview = true
+explicit-preview-rules = true
+select = [
+    "B",  # flake8-bugbear
+    "D",  # pydocstyle
+    "E",  # pycodestyle
+    "F",  # pyflakes
+    "I",  # isort
+    "EM",  # flake8-errmsg
+    "T20",  # flake8-print
+    "RET",  # flake8-return
+    "ANN",  # flake8-annotations
+    "DOC201",  # pydoclint: return value not documented in docstring
+    "DOC202",  # pydoclint: docstring documents a return value the function lacks
+    "DOC402",  # pydoclint: yield value not documented in docstring
+    "DOC501",  # pydoclint: raised exception not documented in docstring
+    "DOC502",  # pydoclint: docstring documents an exception that is not raised
+]
+ignore = [
+    "D100",  # Ignore missing docstring in public module
+    "D104",  # Ignore missing docstring in public package
+    "D203",  # Ignore one blank line before class docstring
+    "D213",  # Ignore multi-line docstring start on second line
+    "D400",  # Ignore missing period in docstring first line (also captured by D415)
+    "D413",  # Ignore missing newline after final section in docstring
+    "ANN002",  # Ignore missing *args type annotations
+    "ANN003",  # Ignore missing **kwargs type annotations
+    "ANN401",  # Ignore Any type annotations
+]
+
+[tool.ruff.lint.per-file-ignores]
+# Command-line code: printing is its output, not leftover debugging.
+"src/skill_evals/{cli,run,score,init,mlflow_server}.py" = ["T201"]
+"tests/record_trace.py" = ["T201"]
+"src/skill_evals/harnesses/codex/tracing/files/home/.mlflow-codex/setup_tracing.py" = [
+    "T201",
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+# Pull request titles follow Conventional Commits (feat:, fix:, docs:, ...), and squash
+# merges make them the commit messages. `cz bump` reads them to pick the next version.
+[tool.commitizen]
+name = "cz_conventional_commits"
+version_provider = "uv"
+tag_format = "v$version"

skill_evals-0.1.0/src/skill_evals/__init__.py

@@ -0,0 +1,7 @@
+"""Evaluate whether coding agents load and follow skills, in Docker Sandboxes.
+
+The `skill-evals` command (see cli.py) runs evals in sandboxes (run.py) and scores
+their MLflow traces (score.py, with the scorers in scorers/ and the per-harness trace
+readers in traces/). What it runs comes from the project that .skill-evals.yaml
+describes (config.py); `skill-evals init` creates one with the shipped harnesses/.
+"""

skill_evals-0.1.0/src/skill_evals/cli.py

@@ -0,0 +1,162 @@
+"""skill-evals: test whether coding agents load and follow your skills.
+
+    skill-evals init [DIR]        Set up a project (.skill-evals.yaml + folders)
+    skill-evals evaluate          Run the evals, then score them (run + score)
+    skill-evals run               Run evals in sandboxes
+    skill-evals score             Score runs' traces; check thresholds
+    skill-evals validate          Check the project's eval and harness files
+    skill-evals mlflow            Open the MLflow server (UI) on the project's store
+
+run, score and evaluate start an MLflow server for their duration if none is up.
+"""
+
+import argparse
+import sys
+from collections.abc import Callable
+from pathlib import Path
+
+from skill_evals import init, mlflow_server, run, score
+from skill_evals.config import (
+    discover_evals,
+    discover_harnesses,
+    eval_cases,
+    load_harness,
+    use_config,
+)
+
+# A subcommand's function: takes the parsed options, returns an exit code (or, like
+# `run`, something else that counts as success).
+Handler = Callable[[argparse.Namespace], object]
+
+
+def validate(args: argparse.Namespace) -> int:
+    """Validate every eval file and harness.yaml.
+
+    Args:
+        args: The parsed options; `validate` has none of its own.
+
+    Returns:
+        The exit code, 0 (invalid files stop with a message instead).
+    """
+    del args  # Every subcommand takes the parsed options; this one has none
+    evals, harnesses = discover_evals(), discover_harnesses()
+    for name in evals:
+        eval_cases(name)  # Validates the file and resolves fixtures and scorers
+    for name in harnesses:
+        load_harness(name)
+    print(f"valid: {len(evals)} eval(s), {len(harnesses)} harness(es)")
+    return 0
+
+
+def evaluate(args: argparse.Namespace) -> int:
+    """Run the selected evals, then score exactly that run.
+
+    Args:
+        args: The parsed options of `run` plus the threshold options of `score`.
+
+    Returns:
+        The exit code of the scoring: 1 if a threshold was missed, else 0.
+    """
+    args.stamp = run.main(args)
+    return score.main(args)
+
+
+def serve(args: argparse.Namespace) -> int:
+    """Run the MLflow server in the foreground.
+
+    Args:
+        args: The parsed options; `mlflow` has none of its own.
+
+    Returns:
+        The exit code, 0.
+    """
+    del args  # Every subcommand takes the parsed options; this one has none
+    mlflow_server.serve()
+    return 0
+
+
+def _with_mlflow(handler: Handler) -> Handler:
+    """Wrap a subcommand so an MLflow server runs while it does.
+
+    Args:
+        handler: The subcommand's function, taking the parsed options.
+
+    Returns:
+        A function with the same signature.
+    """
+
+    def wrapped(args: argparse.Namespace) -> int:
+        """Run the wrapped subcommand inside `mlflow_server.running`.
+
+        Args:
+            args: The parsed options.
+
+        Returns:
+            The subcommand's exit code; a non-integer result counts as 0.
+        """
+        with mlflow_server.running():
+            result = handler(args)
+        return result if isinstance(result, int) else 0
+
+    return wrapped
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the command-line parser with every subcommand.
+
+    Returns:
+        The parser; each subcommand sets `handler`.
+    """
+    fmt = argparse.RawDescriptionHelpFormatter
+    ap = argparse.ArgumentParser(
+        prog="skill-evals", description=__doc__, formatter_class=fmt
+    )
+    ap.add_argument(
+        "--config",
+        type=Path,
+        help="the project's .skill-evals.yaml (default: the nearest one above the "
+        "current folder)",
+    )
+    sub = ap.add_subparsers(required=True, metavar="COMMAND")
+
+    p = sub.add_parser(
+        "init", help="set up a project", description=init.__doc__, formatter_class=fmt
+    )
+    p.add_argument(
+        "dir", nargs="?", default="skill-evals", help="folder (default: skill-evals)"
+    )
+    p.set_defaults(handler=init.main)
+
+    p = sub.add_parser(
+        "evaluate", help="run, then score that run", description=evaluate.__doc__
+    )
+    run.add_arguments(p)
+    score.add_threshold_arguments(p)
+    p.set_defaults(handler=_with_mlflow(evaluate))
+
+    p = sub.add_parser(
+        "run", help="run evals in sandboxes", description=run.__doc__, formatter_class=fmt
+    )
+    run.add_arguments(p)
+    p.set_defaults(handler=_with_mlflow(run.main))
+
+    p = sub.add_parser(
+        "score", help="score runs' traces", description=score.__doc__, formatter_class=fmt
+    )
+    score.add_arguments(p)
+    p.set_defaults(handler=_with_mlflow(score.main))
+
+    p = sub.add_parser("validate", help="check eval and harness files")
+    p.set_defaults(handler=validate)
+
+    p = sub.add_parser("mlflow", help="open the MLflow server (UI) in the foreground")
+    p.set_defaults(handler=serve)
+    return ap
+
+
+def main() -> None:
+    """Parse the command line and run the chosen subcommand."""
+    args = build_parser().parse_args()
+    use_config(args.config)
+    code = args.handler(args)
+    sys.exit(code if isinstance(code, int) else 0)