patchground 0.1.0__py3-none-any.whl

patchground/verify/reference.py

@@ -0,0 +1,158 @@
+"""Apply ``reference.patch`` to a temporary copy of the starting workspace with host git.
+
+``git apply`` here is not a challenge command: it only edits files in a copy that lives in the
+system temp directory, outside any repository. Git must never read configuration the challenge
+controls: a ``.git`` inside the workspace could define ``filter.*.clean``/``smudge`` commands that
+``.gitattributes`` would make git run on the host. So the copy never contains ``.git`` entries
+(:func:`copy_workspace`), and git runs with ``GIT_DIR`` pointing at a fresh, empty bare repository
+created next to the copy (no discovery at all), with ``GIT_CEILING_DIRECTORIES`` as a second fence,
+system and global config and system attributes ignored, and hooks, fsmonitor and the attributes
+file disabled. ``--unsafe-paths`` is never passed, so the patch cannot touch anything outside the
+copy.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import stat
+import subprocess
+import tempfile
+from pathlib import Path
+
+from patchground import safeio
+from patchground.verify.errors import VerifyEnvironmentError
+
+GIT_TIMEOUT_SECONDS = 60.0
+
+
+class PatchError(Exception):
+    """``git apply`` rejected the patch; ``stderr`` is git's explanation."""
+
+    def __init__(self, stderr: str) -> None:
+        super().__init__(stderr)
+        self.stderr = stderr
+
+
+def snapshot(directory: Path) -> dict[str, str]:
+    """Map relative POSIX path -> sha256 of the content for every non-directory entry.
+
+    Symlinks are recorded by their target text and never followed.
+    """
+    entries: dict[str, str] = {}
+    for current, dirnames, filenames in directory.walk(follow_symlinks=False):
+        names = filenames + [d for d in dirnames if (current / d).is_symlink()]
+        for name in names:
+            path = current / name
+            rel = path.relative_to(directory).as_posix()
+            if path.is_symlink():
+                entries[rel] = "L" + hashlib.sha256(os.fsencode(path.readlink())).hexdigest()
+            elif stat.S_ISREG(path.lstat().st_mode):
+                entries[rel] = hashlib.sha256(path.read_bytes()).hexdigest()
+            else:
+                entries[rel] = "special"
+    return entries
+
+
+def _is_git_entry(name: str) -> bool:
+    return name.casefold() == ".git"
+
+
+def _ignore_git(_directory: str, names: list[str]) -> set[str]:
+    return {name for name in names if _is_git_entry(name)}
+
+
+def copy_workspace(source: Path, target: Path) -> None:
+    """Copy ``source`` to the new directory ``target`` without any ``.git`` entry (any case/depth).
+
+    Symlinks are copied as symlinks. Raises :class:`VerifyEnvironmentError` if the copy fails.
+    """
+    try:
+        shutil.copytree(source, target, symlinks=True, ignore=_ignore_git)
+    except (OSError, shutil.Error) as exc:
+        raise VerifyEnvironmentError(
+            f"could not copy the workspace to a temporary folder: {exc}"
+        ) from exc
+
+
+_GIT_HARDENING = (
+    "-c",
+    f"core.hooksPath={os.devnull}",
+    "-c",
+    "core.fsmonitor=false",
+    "-c",
+    f"core.attributesFile={os.devnull}",
+)
+
+
+def _git_env(temp_root: Path) -> dict[str, str]:
+    return {
+        "PATH": os.defpath,
+        "HOME": str(temp_root),
+        "LC_ALL": "C",
+        "GIT_CEILING_DIRECTORIES": str(temp_root.parent),
+        "GIT_CONFIG_NOSYSTEM": "1",
+        "GIT_CONFIG_GLOBAL": os.devnull,
+        "GIT_ATTR_NOSYSTEM": "1",
+        "GIT_TERMINAL_PROMPT": "0",
+    }
+
+
+def _git(*inputs: Path) -> str:
+    return safeio.resolve_git(*inputs, error=VerifyEnvironmentError)
+
+
+def apply_patch(
+    patch: Path, target: Path, temp_root: Path, *, inputs: tuple[Path, ...] = ()
+) -> None:
+    """Apply ``patch`` (paths relative to ``target``, ``-p1``) inside ``target``.
+
+    ``temp_root`` is the temporary directory that contains ``target``; an empty bare repository
+    is created there (outside ``target``) and used as ``GIT_DIR``, so git never discovers or
+    reads any other repository, including a ``.git`` inside ``target``. Raises
+    :class:`PatchError` if git rejects the patch and :class:`VerifyEnvironmentError` if git is
+    missing or cannot run.
+    """
+    git = _git(patch.parent, target, temp_root, *inputs)
+    target = target.resolve()
+    env = _git_env(temp_root)
+    try:
+        git_dir = Path(tempfile.mkdtemp(prefix="gitdir-", dir=temp_root)).resolve()
+    except OSError as exc:
+        raise VerifyEnvironmentError(f"could not create a temporary git directory: {exc}") from exc
+    if git_dir.is_relative_to(target):
+        raise ValueError("temp_root must not lie inside the target")
+    init = _run_git(
+        [git, *_GIT_HARDENING, "init", "--bare", "-q", "--template=", str(git_dir)], temp_root, env
+    )
+    if init.returncode != 0:
+        raise VerifyEnvironmentError("could not initialize the isolated reference repository")
+    env = {**env, "GIT_DIR": str(git_dir), "GIT_WORK_TREE": str(target)}
+    patch_arg = str(patch.resolve())
+    for argv in (
+        [git, *_GIT_HARDENING, "apply", "--check", "-p1", patch_arg],
+        [git, *_GIT_HARDENING, "apply", "-p1", "--whitespace=nowarn", patch_arg],
+    ):
+        completed = _run_git(argv, target, env)
+        if completed.returncode != 0:
+            raise PatchError(f"git apply exited {completed.returncode}")
+
+
+def _run_git(argv: list[str], cwd: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
+    try:
+        return subprocess.run(
+            argv,
+            cwd=cwd,
+            env=env,
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            text=True,
+            errors="replace",
+            timeout=GIT_TIMEOUT_SECONDS,
+            check=False,
+        )
+    except OSError as exc:
+        raise VerifyEnvironmentError(f"could not run git: {exc}") from exc
+    except subprocess.TimeoutExpired as exc:
+        raise VerifyEnvironmentError(f"git did not finish within {GIT_TIMEOUT_SECONDS:g}s") from exc

patchground/verify/result.py

@@ -0,0 +1,199 @@
+"""Structured results of a validation: per-step results and the overall report.
+
+Everything here is plain data with a JSON form (``to_dict``). The JSON form never carries file
+contents, the reference patch or logs of runs that include hidden checks: only test ids and
+statuses, counts, and short human-readable details.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+
+from patchground.sandbox import RunOutcome, RunResult, TestStatus
+from patchground.spec import Challenge, ManifestIssue
+
+
+class StepName(StrEnum):
+    """The verification steps of ``patchground-contract-verification``, in order."""
+
+    MANIFEST = "manifest"
+    BUILD = "build"
+    BROKEN_FAILS = "broken_fails"
+    REFERENCE_PASSES = "reference_passes"
+    DETERMINISM = "determinism"
+    ANTI_GAMING = "anti_gaming"
+
+
+class StepStatus(StrEnum):
+    PASSED = "passed"
+    FAILED = "failed"
+    NOT_RUN = "not_run"
+
+
+def utc_now() -> str:
+    """Current UTC time as ISO 8601 with millisecond precision and a ``Z`` suffix."""
+    return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+@dataclass(frozen=True)
+class TestOutcome:
+    """One test of a run: its id and status only."""
+
+    __test__ = False  # not a pytest test class
+
+    id: str
+    status: TestStatus
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"id": self.id, "status": self.status.value}
+
+
+@dataclass(frozen=True)
+class RunSummary:
+    """One sandbox run as recorded by validation (no log text)."""
+
+    label: str
+    outcome: RunOutcome
+    tests: tuple[TestOutcome, ...]
+
+    @classmethod
+    def from_result(cls, label: str, result: RunResult) -> RunSummary:
+        return cls(label, result.outcome, tuple(TestOutcome(t.id, t.status) for t in result.tests))
+
+    def count(self, status: TestStatus) -> int:
+        return sum(1 for t in self.tests if t.status is status)
+
+    def ids(self) -> list[str]:
+        return [t.id for t in self.tests]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "label": self.label,
+            "outcome": self.outcome.value,
+            "counts": {
+                "passed": self.count(TestStatus.PASSED),
+                "failed": self.count(TestStatus.FAILED),
+                "errored": self.count(TestStatus.ERRORED),
+                "skipped": self.count(TestStatus.SKIPPED),
+            },
+            "tests": [t.to_dict() for t in self.tests],
+        }
+
+
+@dataclass(frozen=True)
+class StepResult:
+    """Result of one step.
+
+    ``issues``/``issue_lines`` are set for a failed ``manifest`` step (``issue_lines`` are the
+    ``file:line: field: message`` renderings, relative to the working directory when possible);
+    ``log_tail`` holds the end of the build log for a failed ``build`` step. ``notes`` are
+    sandbox explanations (``[patchground]`` lines) for inconclusive runs.
+    """
+
+    name: StepName
+    status: StepStatus
+    detail: str
+    started_at: str | None = None
+    finished_at: str | None = None
+    runs: tuple[RunSummary, ...] = ()
+    issues: tuple[ManifestIssue, ...] = ()
+    issue_lines: tuple[str, ...] = ()
+    log_tail: str | None = None
+    notes: tuple[str, ...] = ()
+
+    def to_record(self) -> dict[str, Any]:
+        """The shape stored in ``verification.json``."""
+        return {
+            "name": self.name.value,
+            "status": self.status.value,
+            "detail": self.detail,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "runs": [run.to_dict() for run in self.runs],
+        }
+
+    def to_dict(self) -> dict[str, Any]:
+        """The record shape plus the failure diagnostics that are set."""
+        data = self.to_record()
+        if self.issues:
+            data["issues"] = [
+                {
+                    "file": str(issue.file),
+                    "line": issue.line,
+                    "field": issue.field,
+                    "message": issue.message,
+                    "text": text,
+                }
+                for issue, text in zip(self.issues, self.issue_lines, strict=True)
+            ]
+        if self.log_tail is not None:
+            data["log_tail"] = self.log_tail
+        if self.notes:
+            data["notes"] = list(self.notes)
+        return data
+
+
+@dataclass(frozen=True)
+class ValidationReport:
+    """Outcome of :func:`patchground.verify.validate_challenge`."""
+
+    folder: Path
+    steps: tuple[StepResult, ...]
+    validated_at: str
+    challenge: Challenge | None = None
+    image_id: str | None = None
+    verification_file: Path | None = field(default=None)
+    removed_verification_file: Path | None = field(default=None)
+
+    @property
+    def valid(self) -> bool:
+        """True when no step failed and at least one step ran."""
+        return self.failed_step is None and any(
+            step.status is StepStatus.PASSED for step in self.steps
+        )
+
+    @property
+    def failed_step(self) -> StepName | None:
+        for step in self.steps:
+            if step.status is StepStatus.FAILED:
+                return step.name
+        return None
+
+    def step(self, name: StepName) -> StepResult:
+        for step in self.steps:
+            if step.name is name:
+                return step
+        raise KeyError(name)
+
+    def to_dict(self) -> dict[str, Any]:
+        failed = self.failed_step
+        return {
+            "valid": self.valid,
+            "failed_step": failed.value if failed is not None else None,
+            "steps": [step.to_dict() for step in self.steps],
+            "verification_file": (
+                str(self.verification_file) if self.verification_file is not None else None
+            ),
+            "removed_verification_file": (
+                str(self.removed_verification_file)
+                if self.removed_verification_file is not None
+                else None
+            ),
+        }
+
+
+def run_label(*parts: str) -> str:
+    return "+".join(parts)
+
+
+def format_ids(ids: Sequence[str], limit: int = 10) -> str:
+    """Comma-separated ids, truncated after ``limit`` with a count of the rest."""
+    shown = ", ".join(ids[:limit])
+    if len(ids) > limit:
+        shown += f", … ({len(ids) - limit} more)"
+    return shown

patchground/verify/runners.py

@@ -0,0 +1,47 @@
+"""Recognize the bounded runner recipes without executing authored commands."""
+
+from __future__ import annotations
+
+import re
+import shlex
+from pathlib import PurePath, PurePosixPath
+from typing import Literal
+
+from patchground.runner_assets import VITEST_COMMAND
+from patchground.verify.pytest_adapter import supports_pytest
+
+Runner = Literal["pytest", "vitest"]
+VITEST_EXCLUDED_PARTS = frozenset({"node_modules", ".git"})
+_SAFE_MODULE = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_./-]*")
+
+
+def is_vitest_path(path: PurePath) -> bool:
+    return path.name.endswith(".test.ts") and not VITEST_EXCLUDED_PARTS.intersection(path.parts)
+
+
+def vitest_identity(module: str, name: str) -> str:
+    path = PurePosixPath(module)
+    identity = f"{module}::{name}"
+    if (
+        not _SAFE_MODULE.fullmatch(module)
+        or not is_vitest_path(path)
+        or path.is_absolute()
+        or ".." in path.parts
+        or any(part.startswith("-") for part in path.parts)
+        or not name
+        or not identity.isprintable()
+        or len(identity) > 240
+    ):
+        raise ValueError("TypeScript test identity cannot be represented safely")
+    return identity
+
+
+def runner_for(command: str) -> Runner | None:
+    if supports_pytest(command):
+        return "pytest"
+    try:
+        if shlex.split(command) == shlex.split(VITEST_COMMAND):
+            return "vitest"
+    except ValueError:
+        pass
+    return None

patchground/verify/tampering.py

@@ -0,0 +1,106 @@
+"""Bounded pytest test-module mutations in disposable editable layers only."""
+
+from __future__ import annotations
+
+import ast
+from collections import Counter
+from pathlib import Path
+from typing import Literal
+
+from patchground.verify.pytest_adapter import (
+    UnsupportedPytest,
+    is_test_filename,
+    iter_test_definitions,
+    supports_pytest,
+)
+from patchground.verify.pytest_adapter import (
+    parse_module as parse_pytest_module,
+)
+from patchground.verify.runners import Runner, is_vitest_path, vitest_identity
+from patchground.verify.vitest_adapter import UnsupportedVitest, parse_module, skip_module
+
+__all__ = [
+    "Mutation",
+    "UnsupportedTampering",
+    "mutate",
+    "supports_pytest",
+    "test_modules",
+    "vitest_identities",
+]
+
+Mutation = Literal["empty", "deleted", "skipped"]
+
+
+class UnsupportedTampering(Exception):
+    """No supported mutation can be established; never report this as verified."""
+
+
+def _syntax(path: Path) -> ast.Module:
+    try:
+        return parse_pytest_module(path)
+    except UnsupportedPytest as exc:
+        raise UnsupportedTampering("editable modules are not supported Python source") from exc
+
+
+def _vitest_modules(layers: list[Path]) -> list[tuple[Path, str]]:
+    return [
+        (path, path.relative_to(layer).as_posix())
+        for layer in layers
+        for path in sorted(layer.rglob("*.test.ts"))
+        if is_vitest_path(path.relative_to(layer))
+    ]
+
+
+def vitest_identities(layers: list[Path]) -> Counter[str]:
+    """Expected editable identities, checked against actual collection before mutation."""
+    try:
+        return Counter(
+            vitest_identity(relative, name)
+            for path, relative in _vitest_modules(layers)
+            for name in parse_module(path).definitions
+        )
+    except (UnsupportedVitest, ValueError) as exc:
+        raise UnsupportedTampering(
+            "TypeScript test identities cannot be established safely"
+        ) from exc
+
+
+def test_modules(layers: list[Path], *, runner: Runner = "pytest") -> list[Path]:
+    """Select conventional modules with static tests, preserving helper-only modules."""
+    if runner == "vitest":
+        try:
+            return [path for path, _ in _vitest_modules(layers) if parse_module(path).definitions]
+        except UnsupportedVitest as exc:
+            raise UnsupportedTampering("unsupported TypeScript test module") from exc
+    candidates = sorted(
+        path for layer in layers for path in layer.rglob("*.py") if is_test_filename(path.name)
+    )
+    return [
+        path for path in candidates if next(iter_test_definitions(_syntax(path)), None) is not None
+    ]
+
+
+def mutate(modules: list[Path], mutation: Mutation, *, runner: Runner = "pytest") -> None:
+    for path in modules:
+        if mutation == "deleted":
+            path.unlink()
+        elif mutation == "skipped" and runner == "vitest":
+            try:
+                skip_module(path)
+            except UnsupportedVitest as exc:
+                raise UnsupportedTampering("unsupported TypeScript skip mutation") from exc
+        elif mutation == "skipped":
+            syntax = _syntax(path)
+            names = {node.id for node in ast.walk(syntax) if isinstance(node, ast.Name)}
+            alias = "_patchground_anti_gaming_pytest"
+            while alias in names:
+                alias += "_"
+            suffix = (
+                f"\nimport pytest as {alias}\n"
+                "pytestmark = globals().get('pytestmark', [])\n"
+                "pytestmark = (list(pytestmark) if isinstance(pytestmark, (list, tuple)) "
+                "else [pytestmark]) + "
+                f"[{alias}.mark.skip(reason='Patchground editable-test probe')]\n"
+            )
+            with path.open("ab") as stream:
+                stream.write(suffix.encode("ascii"))