secondshift 0.1.0b12__py3-none-any.whl

secondshift/repositories/snapshots.py

@@ -0,0 +1,238 @@
+"""Build the files of one commit into a snapshot a run can unpack.
+
+A snapshot is a gzip tar of the commit's tree, built from Git objects directly.
+It does not use `git archive`, because that follows the repository's own
+.gitattributes, and `export-ignore` or `export-subst` could make the files a
+run gets differ from the commit. The same commit always gives the same bytes,
+so a retried run start sends exactly what the first attempt sent.
+"""
+
+import gzip
+import io
+import re
+import subprocess
+import tarfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import IO
+
+from secondshift.config.constants import (
+    MAX_SKILL_FILES,
+    MAX_SKILL_SNAPSHOT_BYTES,
+    MAX_SKILL_UNPACKED_BYTES,
+    MAX_SOURCE_FILES,
+    MAX_SOURCE_SNAPSHOT_BYTES,
+    MAX_SOURCE_UNPACKED_BYTES,
+)
+
+REGULAR_FILE_MODES = {"100644": 0o644, "100755": 0o755}
+SYMLINK_MODE = "120000"
+SUBMODULE_MODE = "160000"
+COMMIT_SHA = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}")
+
+
+class SnapshotError(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True)
+class SnapshotLimits:
+    files: int
+    unpacked_bytes: int
+    snapshot_bytes: int
+    # What the snapshot is of, in error messages.
+    label: str
+    # A repository keeps its links and submodules. A skill from another
+    # repository may hold only regular files.
+    regular_files_only: bool = False
+
+
+REPOSITORY_LIMITS = SnapshotLimits(
+    MAX_SOURCE_FILES,
+    MAX_SOURCE_UNPACKED_BYTES,
+    MAX_SOURCE_SNAPSHOT_BYTES,
+    "The commit",
+)
+SKILL_LIMITS = SnapshotLimits(
+    MAX_SKILL_FILES,
+    MAX_SKILL_UNPACKED_BYTES,
+    MAX_SKILL_SNAPSHOT_BYTES,
+    "The skill",
+    regular_files_only=True,
+)
+
+
+@dataclass(frozen=True)
+class TreeEntry:
+    mode: str
+    object_id: str
+    size: int
+    path: str
+
+
+def tree_entries(git_dir: Path, commit_sha: str, folder: str = "") -> list[TreeEntry]:
+    """Every file, link and submodule in the commit, in Git's order.
+
+    With a folder, only what is inside it, with paths relative to it.
+    """
+    if not COMMIT_SHA.fullmatch(commit_sha):
+        raise SnapshotError("A full commit SHA is required.")
+    tree = f"{commit_sha}:{folder}" if folder else f"{commit_sha}^{{tree}}"
+    output = run_git_bytes(git_dir, "ls-tree", "-r", "-z", "-l", "--full-tree", tree)
+    entries = []
+    for record in output.split(b"\0"):
+        if not record:
+            continue
+        header, path = record.split(b"\t", 1)
+        mode, _, object_id, size = header.decode().split()
+        entries.append(
+            TreeEntry(
+                mode=mode,
+                object_id=object_id,
+                size=0 if size == "-" else int(size),
+                path=path.decode("utf-8", errors="surrogateescape"),
+            )
+        )
+    return entries
+
+
+def write_snapshot(
+    git_dir: Path,
+    commit_sha: str,
+    output: IO[bytes],
+    folder: str = "",
+    limits: SnapshotLimits = REPOSITORY_LIMITS,
+) -> None:
+    """Write the commit's files, or one folder's, to `output` as a
+    reproducible .tar.gz."""
+    entries = tree_entries(git_dir, commit_sha, folder)
+    if len(entries) > limits.files:
+        raise SnapshotError(f"{limits.label} has more than {limits.files:,} files.")
+    if sum(entry.size for entry in entries) > limits.unpacked_bytes:
+        raise SnapshotError(f"{limits.label}'s files are too large to run.")
+    if limits.regular_files_only and (
+        links := [
+            entry.path for entry in entries if entry.mode not in REGULAR_FILE_MODES
+        ]
+    ):
+        raise SnapshotError(
+            f"{limits.label} may hold only regular files. Not allowed: "
+            + ", ".join(links[:5])
+        )
+    limited = LimitedWriter(output, limits.snapshot_bytes, limits.label)
+    with (
+        gzip.GzipFile(
+            fileobj=limited, mode="wb", mtime=0, filename="", compresslevel=6
+        ) as compressed,
+        tarfile.open(
+            fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT
+        ) as archive,
+        BlobReader(git_dir) as blobs,
+    ):
+        for entry in entries:
+            info = tarfile.TarInfo(entry.path)
+            info.mtime = 0
+            if entry.mode in REGULAR_FILE_MODES:
+                info.mode = REGULAR_FILE_MODES[entry.mode]
+                info.size = entry.size
+                archive.addfile(info, blobs.stream(entry.object_id, entry.size))
+                blobs.finish()
+            elif entry.mode == SYMLINK_MODE:
+                info.type = tarfile.SYMTYPE
+                info.mode = 0o777
+                info.linkname = blobs.read(entry.object_id, entry.size).decode(
+                    "utf-8", errors="surrogateescape"
+                )
+                archive.addfile(info)
+            elif entry.mode == SUBMODULE_MODE:
+                # Submodule contents are not fetched. Like a checkout without
+                # them, the folder exists but is empty.
+                info.type = tarfile.DIRTYPE
+                info.mode = 0o755
+                archive.addfile(info)
+            else:
+                raise SnapshotError(f"Unsupported Git file mode {entry.mode}.")
+
+
+class LimitedWriter(io.RawIOBase):
+    def __init__(self, output: IO[bytes], limit: int, label: str) -> None:
+        self.output = output
+        self.limit = limit
+        self.remaining = limit
+        self.label = label
+
+    def writable(self) -> bool:
+        return True
+
+    def write(self, data: bytes) -> int:  # type: ignore[override]
+        self.remaining -= len(data)
+        if self.remaining < 0:
+            raise SnapshotError(
+                f"{self.label}'s files are larger than "
+                f"{self.limit // (1024 * 1024)} MiB compressed."
+            )
+        return self.output.write(data)
+
+
+class BlobReader:
+    """Reads blobs through one `git cat-file --batch` process."""
+
+    def __init__(self, git_dir: Path) -> None:
+        self.git_dir = git_dir
+        self.process: subprocess.Popen[bytes] | None = None
+
+    def __enter__(self) -> "BlobReader":
+        self.process = subprocess.Popen(
+            ["git", "-C", str(self.git_dir), "cat-file", "--batch"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+        )
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        if self.process is None:
+            return
+        if self.process.stdin:
+            self.process.stdin.close()
+        self.process.kill()
+        self.process.wait(timeout=10)
+        if self.process.stdout:
+            self.process.stdout.close()
+
+    def stream(self, object_id: str, size: int) -> IO[bytes]:
+        """The blob's content. Read exactly `size` bytes, then call finish()."""
+        process = self.process
+        if process is None or process.stdin is None or process.stdout is None:
+            raise SnapshotError("The blob reader is not open.")
+        process.stdin.write(f"{object_id}\n".encode())
+        process.stdin.flush()
+        header = process.stdout.readline().decode().split()
+        if len(header) != 3 or header[1] != "blob" or int(header[2]) != size:
+            raise SnapshotError(f"Git could not read object {object_id}.")
+        return process.stdout
+
+    def finish(self) -> None:
+        # cat-file ends each object with a newline.
+        if self.process and self.process.stdout:
+            self.process.stdout.read(1)
+
+    def read(self, object_id: str, size: int) -> bytes:
+        content = self.stream(object_id, size).read(size)
+        self.finish()
+        return content
+
+
+def run_git_bytes(git_dir: Path, *arguments: str) -> bytes:
+    try:
+        completed = subprocess.run(
+            ["git", "-C", str(git_dir), *arguments],
+            capture_output=True,
+            check=False,
+            timeout=120,
+        )
+    except subprocess.TimeoutExpired as error:
+        raise SnapshotError("Git timed out while reading the commit.") from error
+    if completed.returncode:
+        raise SnapshotError("Git could not read the commit.")
+    return completed.stdout

secondshift/repositories/sources.py

@@ -0,0 +1,256 @@
+"""The snapshot of a commit's files that a run gets.
+
+Snapshots are built on the control plane, cached by repository and commit, and
+uploaded to the runner when a run starts. The cache can be deleted at any time.
+A missing snapshot is built again from the mirror, or the local folder. The
+cache is kept under SOURCE_CACHE_MAX_BYTES, least recently used first.
+"""
+
+import hashlib
+import os
+import re
+import shutil
+import tempfile
+import time
+from collections.abc import Collection
+from dataclasses import dataclass
+from pathlib import Path
+
+from django.conf import settings
+
+from secondshift.config.constants import COMMIT_SHA_PATTERN
+from secondshift.repositories.dao import active_repository_ids
+from secondshift.repositories.github_access import github_access
+from secondshift.repositories.github_sync import fetch_github_commit
+from secondshift.repositories.mirrors import has_commit, locked_mirror
+from secondshift.repositories.models import Repository
+from secondshift.repositories.snapshots import (
+    REPOSITORY_LIMITS,
+    SnapshotError,
+    SnapshotLimits,
+    write_snapshot,
+)
+
+COMMIT_SHA = re.compile(COMMIT_SHA_PATTERN)
+# A skill from another repository that no run used for this long is removed.
+SKILL_UNUSED_SECONDS = 30 * 24 * 60 * 60
+
+
+@dataclass(frozen=True)
+class SourceSnapshot:
+    path: Path
+    sha256: str
+    size_bytes: int
+
+
+def snapshots_root() -> Path:
+    return settings.SOURCE_CACHE_ROOT / "snapshots"
+
+
+def skill_snapshots_root() -> Path:
+    """Folders of skills from other repositories. See external_skills."""
+    return settings.SOURCE_CACHE_ROOT / "skill-snapshots"
+
+
+def skill_mirrors_root() -> Path:
+    return settings.SOURCE_CACHE_ROOT / "skill-mirrors"
+
+
+def snapshot_path(repository_id: int, commit_sha: str) -> Path:
+    return snapshots_root() / str(repository_id) / f"{commit_sha}.tar.gz"
+
+
+def source_snapshot(repository: Repository, commit_sha: str) -> SourceSnapshot:
+    """The commit's snapshot, built and cached if needed."""
+    if not COMMIT_SHA.fullmatch(commit_sha):
+        raise SnapshotError("A full commit SHA is required.")
+    path = snapshot_path(repository.pk, commit_sha)
+    if path.exists():
+        # Mark it recently used, so cache cleanup removes older ones first.
+        path.touch()
+    else:
+        build_snapshot(repository, commit_sha, path)
+        trim_snapshots(keep=path)
+    return SourceSnapshot(
+        path=path, sha256=file_sha256(path), size_bytes=path.stat().st_size
+    )
+
+
+def build_snapshot(repository: Repository, commit_sha: str, path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    if repository.source_type == Repository.SourceType.LOCAL:
+        if not settings.LOCAL_REPOSITORIES_ENABLED:
+            raise SnapshotError(
+                "Local repositories are available only on a single host."
+            )
+        write_atomically(Path(repository.local_path), commit_sha, path)
+        return
+    with locked_mirror(repository.pk) as mirror:
+        if not has_commit(mirror, commit_sha):
+            fetch_github_commit(github_access(repository), commit_sha, mirror)
+        write_atomically(mirror, commit_sha, path)
+
+
+def write_atomically(
+    git_dir: Path,
+    commit_sha: str,
+    path: Path,
+    folder: str = "",
+    limits: SnapshotLimits = REPOSITORY_LIMITS,
+) -> None:
+    # Two runs of one commit may build at once. Both write the same bytes, and
+    # each replaces the file whole, so neither ever reads a partial snapshot.
+    descriptor, name = tempfile.mkstemp(dir=path.parent, prefix="building-")
+    temporary = Path(name)
+    try:
+        with os.fdopen(descriptor, "wb") as output:
+            write_snapshot(git_dir, commit_sha, output, folder, limits)
+        os.replace(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        while chunk := stream.read(1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+@dataclass
+class CachePrune:
+    snapshots: int = 0
+    mirrors: int = 0
+    bytes: int = 0
+
+    def add(self, path: Path) -> None:
+        self.bytes += tree_size(path)
+        if path.name.endswith(".git"):
+            self.mirrors += 1
+        else:
+            self.snapshots += 1
+
+
+def prune_source_cache(dry_run: bool = False) -> CachePrune:
+    """Remove cached sources no repository needs, then trim snapshots to size.
+
+    Only entries named by a repository ID are touched, and only inside the
+    cache folder. Removed repositories lose their mirror and snapshots.
+    """
+    pruned = CachePrune()
+    active = active_repository_ids()
+    removed = set()
+    for folder in cache_entries(snapshots_root(), suffix=""):
+        if int(folder.name) not in active:
+            removed.add(int(folder.name))
+            for snapshot in folder.glob("*.tar.gz"):
+                pruned.add(snapshot)
+            if not dry_run:
+                shutil.rmtree(folder)
+    for mirror in cache_entries(settings.SOURCE_CACHE_ROOT / "mirrors", ".git"):
+        repository_id = int(mirror.name.removesuffix(".git"))
+        if repository_id not in active:
+            pruned.add(mirror)
+            if not dry_run:
+                with locked_mirror(repository_id) as path:
+                    shutil.rmtree(path)
+    remove_stale_builds(pruned, dry_run)
+    trim_snapshots(pruned, dry_run=dry_run, skip=removed)
+    prune_unused_skills(pruned, dry_run)
+    return pruned
+
+
+def prune_unused_skills(pruned: CachePrune, dry_run: bool) -> None:
+    """Remove skills from other repositories that no run used for a while."""
+    cutoff = time.time() - SKILL_UNUSED_SECONDS
+    for path in [
+        *cached_paths(skill_snapshots_root(), "*.tar.gz"),
+        *cached_paths(skill_mirrors_root(), "*.git"),
+    ]:
+        if path.stat().st_mtime >= cutoff:
+            continue
+        pruned.add(path)
+        if dry_run:
+            continue
+        if path.is_dir():
+            shutil.rmtree(path, ignore_errors=True)
+        else:
+            path.unlink(missing_ok=True)
+
+
+def cached_paths(root: Path, pattern: str) -> list[Path]:
+    if not root.is_dir():
+        return []
+    return [path for path in root.glob(pattern) if not path.is_symlink()]
+
+
+def trim_snapshots(
+    pruned: CachePrune | None = None,
+    dry_run: bool = False,
+    keep: Path | None = None,
+    skip: Collection[int] = (),
+) -> CachePrune:
+    """Remove the least recently used snapshots past SOURCE_CACHE_MAX_BYTES."""
+    pruned = pruned or CachePrune()
+    files = sorted(
+        (
+            path
+            for folder in cache_entries(snapshots_root(), suffix="")
+            if int(folder.name) not in skip
+            for path in folder.glob("*.tar.gz")
+            if path.is_file() and not path.is_symlink()
+        ),
+        key=lambda path: path.stat().st_mtime,
+    )
+    total = sum(path.stat().st_size for path in files)
+    for path in files:
+        if total <= settings.SOURCE_CACHE_MAX_BYTES:
+            break
+        if path == keep:
+            continue
+        size = path.stat().st_size
+        total -= size
+        pruned.snapshots += 1
+        pruned.bytes += size
+        if not dry_run:
+            path.unlink(missing_ok=True)
+    return pruned
+
+
+def remove_stale_builds(pruned: CachePrune, dry_run: bool) -> None:
+    """Remove partial snapshots left by a build that crashed."""
+    for folder in cache_entries(snapshots_root(), suffix=""):
+        for path in folder.glob("building-*"):
+            if path.is_file() and not path.is_symlink() and is_stale(path):
+                pruned.bytes += path.stat().st_size
+                if not dry_run:
+                    path.unlink(missing_ok=True)
+
+
+def cache_entries(root: Path, suffix: str) -> list[Path]:
+    """Folders in the cache named `<repository id><suffix>`, never links."""
+    if not root.is_dir():
+        return []
+    return [
+        path
+        for path in root.iterdir()
+        if path.name.removesuffix(suffix).isdigit()
+        and path.name.endswith(suffix)
+        and path.is_dir()
+        and not path.is_symlink()
+    ]
+
+
+def is_stale(path: Path) -> bool:
+    return path.stat().st_mtime < time.time() - 60 * 60
+
+
+def tree_size(path: Path) -> int:
+    if path.is_file():
+        return path.stat().st_size
+    return sum(
+        (Path(root) / name).lstat().st_size
+        for root, _, names in os.walk(path)
+        for name in names
+    )

secondshift/repositories/status.py

@@ -0,0 +1,67 @@
+"""What a repository's sync state is, from what its last syncs saved.
+
+A sync saves `checked_sha`, the commit it read, and `last_sync_error` when that
+commit's configuration was broken. A good sync also saves `config_sha`, the
+commit jobs run from. New commits on the branch are reported apart from the
+state, because runs keep using the last good sync.
+"""
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+from secondshift.repositories.models import Repository
+
+
+class ValidationStatus(StrEnum):
+    VALID = "valid"
+    WARNING = "warning"
+    ERROR = "error"
+
+
+class SyncState(StrEnum):
+    NEVER_SYNCED = "never_synced"
+    # A configuration problem before any good sync. Setup is not finished.
+    SETUP_PROBLEM = "setup_problem"
+    # A configuration problem after a good sync. Jobs keep the last good one.
+    BROKEN = "broken"
+    SYNCED = "synced"
+
+
+# How each state reads. A state with no message shows the problem itself.
+STATE_STATUS: dict[SyncState, tuple[ValidationStatus, str]] = {
+    SyncState.NEVER_SYNCED: (
+        ValidationStatus.WARNING,
+        "Sync the repository before enabling schedules.",
+    ),
+    SyncState.SETUP_PROBLEM: (ValidationStatus.WARNING, ""),
+    SyncState.BROKEN: (ValidationStatus.ERROR, ""),
+    SyncState.SYNCED: (ValidationStatus.VALID, "Repository synchronized."),
+}
+
+
+@dataclass(frozen=True)
+class SourceStatus:
+    status: ValidationStatus
+    message: str
+    needs_sync: bool
+
+
+def sync_state(repository: Repository, problem: str) -> SyncState:
+    """`problem` is the configuration problem at the checked commit, if any."""
+    if problem:
+        return SyncState.BROKEN if repository.config_sha else SyncState.SETUP_PROBLEM
+    return SyncState.SYNCED if repository.config_sha else SyncState.NEVER_SYNCED
+
+
+def source_status(
+    repository: Repository, head: str | None, problem: str
+) -> SourceStatus:
+    """The status of a repository whose branch is at `head`, None when unknown."""
+    status, message = STATE_STATUS[sync_state(repository, problem)]
+    return SourceStatus(
+        status=status,
+        message=message or problem,
+        # Before the first sync there is nothing to be behind.
+        needs_sync=bool(head and repository.checked_sha)
+        and head != repository.checked_sha,
+    )

secondshift/runner/README.md

@@ -0,0 +1,65 @@
+# Runner service
+
+The runner is a small, conventional Django deployment with its own settings,
+installed apps, database, credentials, and network policy. Reusing Django gives
+it models, generated migrations, transactions, validation, and a typed Ninja
+API without giving it access to the control-plane applications.
+
+Only the runner deployment receives `/var/run/docker.sock`. The web, worker,
+gateway, and runner use separate settings and database credentials. The runner
+does not receive provider keys, OAuth credentials, encrypted connection data, or
+the control-plane database URL.
+
+Its API is deliberately narrow:
+
+- `HEAD /v1/sources/{sha256}` asks whether a source snapshot is already here
+- `PUT /v1/sources/{sha256}` uploads a snapshot of one commit's files
+- `POST /v1/sessions` creates a sandbox from one uploaded snapshot
+- `POST /v1/sessions/{id}/commands` executes one idempotent tool call
+- `POST /v1/sessions/{id}/collection` stops the sandbox and returns its file
+  changes and saved files as one archive
+- `DELETE /v1/sessions/{id}` destroys the container and workspace
+- `GET /-/health/` is the only unauthenticated endpoint.
+
+All other routes require `SECONDSHIFT_RUNNER_SERVICE_TOKEN`. Session and command
+state live in runner-owned Django models and a runner-only database.
+
+The runner keeps no repositories. When a run starts, the control plane uploads a
+snapshot of the run's commit, named by its SHA-256. The runner checks the bytes
+against that name, unpacks them into the session workspace with size, file
+count and entry type limits, and deletes snapshots an hour after their last
+use. The runner never accepts a checkout path or clone credential.
+
+Production runners must:
+
+- verify `runsc` is registered and refuse to start otherwise
+- unpack only the uploaded snapshot named in the session spec
+- accept only server-owned images, mounts and OCI flags
+- give command sandboxes no network and no credentials
+- use a dedicated sandbox per run with bounded resources
+- keep sandbox images free of the control-plane application and Docker client
+- deduplicate tool calls by their stable run and call identifiers
+- send logs and the final result back through an authenticated channel
+- destroy the workspace after the result is acknowledged.
+
+Start the service only after applying its own migrations:
+
+```bash
+DJANGO_SETTINGS_MODULE=secondshift.config.settings.runner uv run python manage.py migrate
+SECONDSHIFT_SERVICE=runner uv run gunicorn --config python:secondshift.config.gunicorn
+```
+
+Required configuration is in `secondshift/config/settings/runner.py`. Production should use
+`runsc`. Standard Docker is reserved for trusted development and single-tenant
+self-hosting. Service startup verifies the selected runtime and fails closed.
+The runner uses the same image as the control plane, with its own command,
+root user override, database and network identity.
+Sandboxes use `SECONDSHIFT_RUNNER_IMAGE` (default: the pinned official
+`python:3.13-slim`, see `secondshift/config/constants.py`). The runner pulls it at startup and
+refuses to start if the pull fails.
+
+Runner and control-plane apps share one test suite:
+
+```bash
+uv run pytest
+```

secondshift/runner/__init__.py

@@ -0,0 +1 @@
+"""Standalone command runner security boundary."""