patchground 0.1.0__py3-none-any.whl

patchground/spec/models.py

@@ -0,0 +1,394 @@
+"""Pydantic v2 models for ``challenge.yaml`` and ``pack.yaml`` v0.1.
+
+These models are the executable form of ``patchground-contract-challenge-format``. If the contract
+and the models disagree, the contract wins and the models are fixed.
+"""
+
+from __future__ import annotations
+
+import re
+from enum import StrEnum
+from typing import Annotated, Any, Literal, Self
+
+from pydantic import (
+    AfterValidator,
+    BaseModel,
+    BeforeValidator,
+    ConfigDict,
+    Field,
+    ValidationError,
+    WithJsonSchema,
+    field_validator,
+    model_validator,
+)
+from pydantic_core import InitErrorDetails, PydanticCustomError
+
+SPEC_VERSION = "0.1"
+
+EXTENSION_PREFIX = "x-"
+"""Keys starting with this prefix are accepted and ignored at every mapping level."""
+
+SLUG_PATTERN = r"^[a-z0-9]+(-[a-z0-9]+)*$"
+# Semantic Versioning 2.0.0, the regex recommended on semver.org.
+SEMVER_PATTERN = (
+    r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)"
+    r"(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
+    r"(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+)
+
+_SLUG_RE = re.compile(SLUG_PATTERN)
+_SEMVER_RE = re.compile(SEMVER_PATTERN)
+
+
+# --------------------------------------------------------------------------------------------------
+# Enums
+# --------------------------------------------------------------------------------------------------
+
+
+class ChallengeType(StrEnum):
+    """Kind of challenge."""
+
+    DEBUG = "debug"
+    FEATURE = "feature"
+    REVIEW = "review"
+    REFACTOR = "refactor"
+
+
+class Difficulty(StrEnum):
+    """Intended difficulty."""
+
+    BEGINNER = "beginner"
+    INTERMEDIATE = "intermediate"
+    ADVANCED = "advanced"
+
+
+class AIPolicy(StrEnum):
+    """Whether the user may use AI assistance."""
+
+    ALLOWED = "allowed"
+    FORBIDDEN = "forbidden"
+
+
+class Provenance(StrEnum):
+    """How the challenge was authored."""
+
+    HANDMADE = "handmade"
+    GENERATED = "generated"
+
+
+RESERVED_CHALLENGE_TYPES = frozenset({"incident", "ai-validation", "design"})
+RESERVED_AI_POLICIES = frozenset({"trap"})
+
+
+def _reject_reserved(reserved: frozenset[str]) -> Any:
+    def check(value: Any) -> Any:
+        if isinstance(value, str) and value in reserved:
+            raise PydanticCustomError(
+                "reserved_value",
+                "'{value}' is reserved and not part of format v0.1",
+                {"value": value},
+            )
+        return value
+
+    return BeforeValidator(check)
+
+
+def _check_spec_version(value: Any) -> Any:
+    if isinstance(value, float | int) and not isinstance(value, bool) and str(value) == "0.1":
+        raise PydanticCustomError(
+            "spec_version_not_string",
+            'spec_version must be the string "0.1"; quote it: spec_version: "0.1"',
+        )
+    return value
+
+
+SpecVersion = Annotated[Literal["0.1"], BeforeValidator(_check_spec_version)]
+
+
+# --------------------------------------------------------------------------------------------------
+# Scalar types
+# --------------------------------------------------------------------------------------------------
+
+
+def _check_not_blank(value: str) -> str:
+    if not value.strip():
+        raise PydanticCustomError("blank_string", "must not be blank")
+    return value
+
+
+def _check_semver(value: str) -> str:
+    if not _SEMVER_RE.match(value):
+        raise PydanticCustomError(
+            "semver", "'{value}' is not a semantic version (e.g. 1.0.0)", {"value": value}
+        )
+    return value
+
+
+def _check_slug(value: str) -> str:
+    if not _SLUG_RE.match(value):
+        raise PydanticCustomError(
+            "slug",
+            "'{value}' is not a kebab-case slug (lowercase letters, digits and single dashes)",
+            {"value": value},
+        )
+    return value
+
+
+def relative_path_problem(value: str, container: str = "challenge folder") -> str | None:
+    """Return why ``value`` is not an acceptable manifest path, or ``None`` if it is.
+
+    Manifest paths are relative POSIX strings that stay inside ``container`` (the challenge folder
+    by default; the workspace for ``expected_paths``) lexically. A trailing ``/`` is allowed.
+    Symlinks are checked against the filesystem by the loader, not here.
+    """
+    if not value or not value.strip():
+        return "path must not be empty"
+    if "\\" in value:
+        return "path must use '/' as separator"
+    if value.startswith("/") or re.match(r"^[A-Za-z]:", value):
+        return f"path must be relative to the {container}"
+    depth = 0
+    for part in value.split("/"):
+        if part in ("", "."):
+            continue
+        if part == "..":
+            depth -= 1
+            if depth < 0:
+                return f"path must stay inside the {container}"
+        else:
+            depth += 1
+    if depth == 0:
+        return f"path must point inside the {container}, not at the {container} itself"
+    return None
+
+
+def _check_relative_path(value: str) -> str:
+    problem = relative_path_problem(value)
+    if problem is not None:
+        raise PydanticCustomError("relative_path", problem)
+    return value
+
+
+def _check_workspace_path(value: str) -> str:
+    problem = relative_path_problem(value, "workspace")
+    if problem is not None:
+        raise PydanticCustomError("relative_path", problem)
+    return value
+
+
+# Custom validators carry the readable messages; WithJsonSchema keeps the exported schema precise.
+NonEmptyStr = Annotated[
+    str,
+    AfterValidator(_check_not_blank),
+    WithJsonSchema({"type": "string", "minLength": 1}),
+]
+Slug = Annotated[
+    str,
+    AfterValidator(_check_slug),
+    WithJsonSchema({"type": "string", "pattern": SLUG_PATTERN}),
+]
+SemVer = Annotated[
+    str,
+    AfterValidator(_check_semver),
+    WithJsonSchema({"type": "string", "pattern": SEMVER_PATTERN}),
+]
+RelPath = Annotated[
+    str,
+    AfterValidator(_check_relative_path),
+    WithJsonSchema(
+        {
+            "type": "string",
+            "minLength": 1,
+            "description": "Relative POSIX path inside the challenge folder "
+            "(no absolute paths, must not escape the folder).",
+        }
+    ),
+]
+
+WorkspacePath = Annotated[
+    str,
+    AfterValidator(_check_workspace_path),
+    WithJsonSchema(
+        {
+            "type": "string",
+            "minLength": 1,
+            "description": "Relative POSIX path inside the workspace (no absolute paths, must not "
+            "escape the workspace; a trailing '/' marks a directory). Need not exist.",
+        }
+    ),
+]
+
+# The models validate the str/int types from YAML without lax coercion of, e.g., "45" -> 45.
+StrictPositiveInt = Annotated[int, Field(strict=True, gt=0)]
+
+
+# --------------------------------------------------------------------------------------------------
+# Base model: unknown keys rejected, ``x-`` keys accepted and ignored
+# --------------------------------------------------------------------------------------------------
+
+
+def _extension_schema(schema: dict[str, Any]) -> None:
+    schema["additionalProperties"] = False
+    schema["patternProperties"] = {"^x-": {}}
+
+
+class ManifestModel(BaseModel):
+    """Shared base: rejects unknown keys except x- extensions, which are dropped."""
+
+    model_config = ConfigDict(
+        extra="forbid",
+        frozen=True,
+        json_schema_extra=_extension_schema,
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def _drop_extension_keys(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            return {
+                key: value
+                for key, value in data.items()
+                if not (isinstance(key, str) and key.startswith(EXTENSION_PREFIX))
+            }
+        return data
+
+
+# --------------------------------------------------------------------------------------------------
+# challenge.yaml
+# --------------------------------------------------------------------------------------------------
+
+
+class Environment(ManifestModel):
+    """Runtime image and test command."""
+
+    dockerfile: RelPath
+    services: list[NonEmptyStr] = Field(default_factory=list)
+    test_command: NonEmptyStr
+
+
+class Checks(ManifestModel):
+    """Visible checks are copied into the workspace; hidden checks never are."""
+
+    visible: RelPath | None = None
+    hidden: RelPath
+
+
+class Author(ManifestModel):
+    """Challenge author."""
+
+    name: NonEmptyStr
+    github: NonEmptyStr | None = None
+
+
+class Debrief(ManifestModel):
+    """Optional interviewer prompts and unweighted coaching areas."""
+
+    questions: RelPath | None = None
+    focus: list[NonEmptyStr] = Field(default_factory=list)
+
+
+class GeneratedBy(ManifestModel):
+    """Model and prompt version of a generated challenge."""
+
+    model: NonEmptyStr
+    prompt_version: NonEmptyStr
+
+
+def _single_error(title: str, loc: tuple[str | int, ...], message: str, value: Any) -> ValueError:
+    """Build a ValidationError that pins ``message`` to ``loc`` (relative to the current model)."""
+    return ValidationError.from_exception_data(
+        title,
+        [InitErrorDetails(type=PydanticCustomError("manifest", message), loc=loc, input=value)],
+    )
+
+
+def _challenge_schema_extra(schema: dict[str, Any]) -> None:
+    _extension_schema(schema)
+    # generated_by is required when provenance is "generated" and forbidden otherwise.
+    schema["if"] = {
+        "properties": {"provenance": {"const": Provenance.GENERATED.value}},
+        "required": ["provenance"],
+    }
+    schema["then"] = {
+        "properties": {"generated_by": {"type": "object"}},
+        "required": ["generated_by"],
+    }
+    schema["else"] = {"properties": {"generated_by": {"type": "null"}}}
+
+
+class ChallengeManifest(ManifestModel):
+    """Manifest of a challenge folder (challenge.yaml, format v0.1)."""
+
+    model_config = ConfigDict(
+        title="Patchground challenge.yaml v0.1",
+        json_schema_extra=_challenge_schema_extra,
+    )
+
+    spec_version: SpecVersion
+    id: Slug
+    version: SemVer
+    title: NonEmptyStr
+    type: Annotated[ChallengeType, _reject_reserved(RESERVED_CHALLENGE_TYPES)]
+    summary: NonEmptyStr
+    languages: Annotated[list[NonEmptyStr], Field(min_length=1)]
+    tags: list[NonEmptyStr] = Field(default_factory=list)
+    difficulty: Difficulty
+    timebox_minutes: StrictPositiveInt
+    ai_policy: Annotated[AIPolicy, _reject_reserved(RESERVED_AI_POLICIES)]
+    environment: Environment
+    checks: Checks
+    solution: RelPath
+    expected_paths: list[WorkspacePath] = Field(default_factory=list)
+    hints: list[NonEmptyStr] = Field(default_factory=list)
+    debrief: Debrief | None = None
+    author: Author
+    content_license: NonEmptyStr = "CC-BY-4.0"
+    provenance: Provenance = Provenance.HANDMADE
+    generated_by: GeneratedBy | None = None
+
+    @model_validator(mode="after")
+    def _check_generated_by(self) -> Self:
+        if self.provenance is Provenance.GENERATED and self.generated_by is None:
+            raise _single_error(
+                type(self).__name__,
+                ("provenance",),
+                "generated_by (model, prompt_version) is required when provenance is 'generated'",
+                self.provenance.value,
+            )
+        if self.provenance is not Provenance.GENERATED and self.generated_by is not None:
+            raise _single_error(
+                type(self).__name__,
+                ("generated_by",),
+                "generated_by is only allowed when provenance is 'generated'",
+                None,
+            )
+        return self
+
+
+# --------------------------------------------------------------------------------------------------
+# pack.yaml
+# --------------------------------------------------------------------------------------------------
+
+
+class PackManifest(ManifestModel):
+    """Manifest of a pack folder (pack.yaml, format v0.1)."""
+
+    model_config = ConfigDict(title="Patchground pack.yaml v0.1")
+
+    spec_version: SpecVersion
+    name: Slug
+    version: SemVer
+    description: NonEmptyStr
+    challenges: Annotated[list[Slug], Field(json_schema_extra={"uniqueItems": True})]
+
+    @field_validator("challenges")
+    @classmethod
+    def _unique_challenges(cls, value: list[str]) -> list[str]:
+        seen: set[str] = set()
+        for index, name in enumerate(value):
+            if name in seen:
+                raise _single_error(
+                    cls.__name__, (index,), f"challenge '{name}' is listed more than once", name
+                )
+            seen.add(name)
+        return value

patchground/spec/schema.py

@@ -0,0 +1,50 @@
+"""Export the JSON Schemas of the v0.1 manifests. The files in ``schema/`` are generated by this.
+
+Usage: ``python -m patchground.spec.schema <out_dir>``
+
+The implementation lives in ``_schema`` so that running this module with ``-m`` does not import it
+twice (the package ``__init__`` re-exports ``export_schemas``).
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from patchground.spec._schema import (
+    GENERATED_COMMENT,
+    JSON_SCHEMA_DIALECT,
+    SCHEMA_BASE_URL,
+    build_schema,
+    export_schemas,
+    render_schema,
+    schema_filename,
+)
+
+__all__ = [
+    "GENERATED_COMMENT",
+    "JSON_SCHEMA_DIALECT",
+    "SCHEMA_BASE_URL",
+    "build_schema",
+    "export_schemas",
+    "main",
+    "render_schema",
+    "schema_filename",
+]
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Command-line entry point: write the schema files into ``out_dir``."""
+    parser = argparse.ArgumentParser(
+        prog="python -m patchground.spec.schema",
+        description="Export the challenge.yaml and pack.yaml JSON Schemas.",
+    )
+    parser.add_argument("out_dir", type=Path, help="directory to write the schema files into")
+    args = parser.parse_args(argv)
+    for path in export_schemas(args.out_dir):
+        print(path)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

patchground/verify/__init__.py

@@ -0,0 +1,29 @@
+"""patchground.verify — see SPEC.md in this folder."""
+
+from patchground.verify.errors import VerifyEnvironmentError
+from patchground.verify.record import VERIFICATION_FILE, read_verification
+from patchground.verify.result import (
+    RunSummary,
+    StepName,
+    StepResult,
+    StepStatus,
+    TestOutcome,
+    ValidationReport,
+)
+from patchground.verify.validate import DEFAULT_LIMITS, DockerSandbox, Sandbox, validate_challenge
+
+__all__ = [
+    "DEFAULT_LIMITS",
+    "VERIFICATION_FILE",
+    "DockerSandbox",
+    "RunSummary",
+    "Sandbox",
+    "StepName",
+    "StepResult",
+    "StepStatus",
+    "TestOutcome",
+    "ValidationReport",
+    "VerifyEnvironmentError",
+    "read_verification",
+    "validate_challenge",
+]

patchground/verify/errors.py

@@ -0,0 +1,10 @@
+"""Exceptions raised by the verify module."""
+
+from __future__ import annotations
+
+
+class VerifyEnvironmentError(Exception):
+    """The host cannot run validation (engine unreachable, git missing, engine failure).
+
+    This is not a verdict on the challenge: no step failed, validation could not proceed.
+    """

patchground/verify/pytest_adapter.py

@@ -0,0 +1,95 @@
+"""Pure pytest recognition shared by anti-gaming probes and regression evidence.
+
+Nothing here executes authored code or touches a container: commands are only tokenised and test
+modules are only parsed. Recognition is deliberately narrow (direct ``pytest`` / ``python -m
+pytest`` invocations, conventional module names, static ``test_*`` functions and ``Test*``
+methods) and anything else is simply not recognised.
+"""
+
+from __future__ import annotations
+
+import ast
+import shlex
+import tokenize
+from collections.abc import Iterator
+from pathlib import Path
+
+MAX_SOURCE_BYTES = 1024 * 1024
+PYTHON_NAMES = ("python", "python3", "python3.12", "python3.13", "python3.14")
+
+TestNode = ast.FunctionDef | ast.AsyncFunctionDef
+
+
+class UnsupportedPytest(Exception):
+    """Source this adapter cannot analyse; messages never carry source text."""
+
+
+def split_pytest_command(command: str) -> tuple[list[str], list[str]] | None:
+    """Return ``(invocation words, arguments)`` for direct pytest, or ``None`` for anything else.
+
+    Shell pipelines, lists, redirections, environment prefixes and wrappers are not recognised.
+    """
+    lexer = shlex.shlex(command, posix=True, punctuation_chars=True)
+    lexer.whitespace_split = True
+    try:
+        words = list(lexer)
+    except ValueError:
+        return None
+    if not words or any(word and all(c in "();<>|&" for c in word) for word in words):
+        return None
+    if words[0] in ("pytest", "pytest-3"):
+        return words[:1], words[1:]
+    if len(words) >= 3 and words[0] in PYTHON_NAMES and words[1:3] == ["-m", "pytest"]:
+        return words[:3], words[3:]
+    return None
+
+
+def supports_pytest(command: str) -> bool:
+    """Recognize direct pytest and python -m pytest invocations, not shell pipelines."""
+    return split_pytest_command(command) is not None
+
+
+def parse_module(path: Path) -> ast.Module:
+    try:
+        with tokenize.open(path) as stream:
+            return ast.parse(stream.read())
+    except (SyntaxError, UnicodeError, LookupError, ValueError) as exc:
+        raise UnsupportedPytest("not supported Python source") from exc
+
+
+def is_test_filename(name: str) -> bool:
+    return name.startswith("test_") or name.endswith("_test.py")
+
+
+def iter_test_definitions(tree: ast.Module) -> Iterator[tuple[str, TestNode]]:
+    """Yield ``(key, node)`` for static tests: ``test_x`` functions and ``TestC::test_x`` methods.
+
+    Duplicate keys are yielded as written; callers decide whether that is ambiguous.
+    """
+    for node in tree.body:
+        if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name.startswith(
+            "test_"
+        ):
+            yield node.name, node
+        elif isinstance(node, ast.ClassDef) and node.name.startswith("Test"):
+            for member in node.body:
+                if isinstance(
+                    member, ast.FunctionDef | ast.AsyncFunctionDef
+                ) and member.name.startswith("test_"):
+                    yield f"{node.name}::{member.name}", member
+
+
+def test_definitions(path: Path) -> dict[str, TestNode]:
+    """Static tests of one module, keyed ``test_x`` or ``TestC::test_x``.
+
+    Raises :class:`UnsupportedPytest` for source over 1 MiB, source that does not parse, and
+    duplicate test names (their identity would be ambiguous). Nothing is executed.
+    """
+    if path.stat().st_size > MAX_SOURCE_BYTES:
+        raise UnsupportedPytest("Python source is too large to analyse")
+    found: dict[str, TestNode] = {}
+    for key, node in iter_test_definitions(parse_module(path)):
+        if key in found:
+            raise UnsupportedPytest("duplicate test names make test identity ambiguous")
+        found[key] = node
+    return found

patchground/verify/record.py

@@ -0,0 +1,94 @@
+"""Write and read ``verification.json`` in a challenge root."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from patchground import __version__
+from patchground.spec.loader import CHALLENGE_MANIFEST
+from patchground.verify.result import ValidationReport
+
+VERIFICATION_FILE = "verification.json"
+FORMAT = "patchground-verification"
+FORMAT_VERSION = 1
+
+
+def verification_document(report: ValidationReport) -> dict[str, Any]:
+    """The ``verification.json`` content for a report whose challenge loaded."""
+    if report.challenge is None:
+        raise ValueError("a verification document needs a loaded challenge")
+    manifest = report.challenge.manifest
+    return {
+        "format": FORMAT,
+        "format_version": FORMAT_VERSION,
+        "patchground_version": __version__,
+        "challenge": {
+            "id": manifest.id,
+            "version": manifest.version,
+            "spec_version": manifest.spec_version,
+        },
+        "image_id": report.image_id,
+        "validated_at": report.validated_at,
+        "steps": [step.to_record() for step in report.steps],
+    }
+
+
+def write_verification(root: Path, document: dict[str, Any]) -> Path:
+    """Atomically write ``document`` to ``root/verification.json`` and return its path."""
+    target = root / VERIFICATION_FILE
+    text = json.dumps(document, indent=2, ensure_ascii=False) + "\n"
+    fd, tmp_name = tempfile.mkstemp(dir=root, prefix=f".{VERIFICATION_FILE}.", suffix=".tmp")
+    tmp = Path(tmp_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            fh.write(text)
+            fh.flush()
+            os.fsync(fh.fileno())
+        tmp.chmod(0o644)
+        tmp.replace(target)
+    except BaseException:
+        with contextlib.suppress(OSError):
+            tmp.unlink()
+        raise
+    return target
+
+
+def remove_verification(folder: Path) -> Path | None:
+    """Delete a stale ``verification.json`` from ``folder``; return its path if one was removed.
+
+    A failed validation must not leave an earlier record claiming the challenge is valid. Only a
+    Patchground record (see :func:`read_verification`) in a folder holding ``challenge.yaml`` is
+    removed; any other file of that name is not ours and is left alone.
+    """
+    folder = Path(folder)
+    target = folder / VERIFICATION_FILE
+    if not (target.is_file() or target.is_symlink()):
+        return None
+    if not (folder / CHALLENGE_MANIFEST).is_file() or read_verification(folder) is None:
+        return None
+    target.unlink()
+    return target
+
+
+def read_verification(folder: Path) -> dict[str, Any] | None:
+    """Return the parsed ``verification.json`` of a challenge folder.
+
+    ``None`` when the file is missing, is not valid JSON, or is not a
+    ``patchground-verification`` document of a supported format version — in every such case
+    the challenge has no usable verification record and needs validating.
+    """
+    path = Path(folder) / VERIFICATION_FILE
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    if data.get("format") != FORMAT or data.get("format_version") != FORMAT_VERSION:
+        return None
+    return data