secondshift 0.1.0b12__py3-none-any.whl

secondshift/runner/collection.py

@@ -0,0 +1,276 @@
+"""What a session leaves behind: its file changes and the files it saved.
+
+When a session that collects changes starts, the runner records the unpacked
+commit in a git folder beside the workspace, outside the sandbox's mounts, so
+the agent cannot change it. Recording costs one write per file, so only runs
+that publish changes ask for it. After the agent finishes, the runner stops the sandbox and compares the
+workspace with that record. The result is one archive:
+
+- ``manifest.json`` describes the changes and the saved files
+- ``changes.patch`` is a binary-safe git diff, if anything changed
+- ``outputs/<name>`` holds each regular file the agent saved in /outputs
+
+Git runs here with no system, global or repository settings that the
+workspace could supply, and never runs hooks, filters or external diff tools.
+"""
+
+import hashlib
+import io
+import json
+import os
+import stat
+import subprocess
+import tarfile
+from pathlib import Path, PurePosixPath
+from typing import IO, Any
+
+from secondshift.config.constants import (
+    MAX_CHANGES_BYTES,
+    MAX_OUTPUT_BYTES,
+    MAX_OUTPUT_FILES,
+)
+from secondshift.runner.protocol import RunnerProtocolError
+
+BASELINE_DIR = "baseline.git"
+OUTPUTS_DIR = "outputs"
+COLLECTION_FILE = "collection.tar.gz"
+MANIFEST_NAME = "manifest.json"
+CHANGES_NAME = "changes.patch"
+# A long list of names helps nobody. The patch itself holds every path.
+MAX_LISTED_CHANGES = 1000
+GIT_TIMEOUT_SECONDS = 300
+NO_CHANGES = {
+    "file_count": 0,
+    "files": [],
+    "size_bytes": 0,
+    "captured": False,
+    "too_large": False,
+}
+
+
+def git(
+    session_root: Path,
+    workspace: Path,
+    *args: str,
+    stdout: IO[bytes] | None = None,
+    work_tree: bool = True,
+) -> subprocess.CompletedProcess[bytes]:
+    git_dir = session_root / BASELINE_DIR
+    command = [
+        "git",
+        # The workspace belongs to the sandbox user, not to the runner.
+        "-c",
+        "safe.directory=*",
+        "-c",
+        "core.hooksPath=/dev/null",
+        "-c",
+        "core.fsmonitor=false",
+        "-c",
+        "core.autocrlf=false",
+        "-c",
+        "core.symlinks=true",
+        f"--git-dir={git_dir}",
+        # git init refuses a work tree. Every other command compares one.
+        *([f"--work-tree={workspace}"] if work_tree else []),
+        *args,
+    ]
+    environment = {
+        "PATH": os.environ.get("PATH", "/usr/bin:/bin"),
+        "HOME": str(git_dir),
+        "LC_ALL": "C",
+        "GIT_CONFIG_NOSYSTEM": "1",
+        "GIT_CONFIG_GLOBAL": os.devnull,
+        "GIT_AUTHOR_NAME": "SecondShift",
+        "GIT_AUTHOR_EMAIL": "runner@secondshift.invalid",
+        "GIT_COMMITTER_NAME": "SecondShift",
+        "GIT_COMMITTER_EMAIL": "runner@secondshift.invalid",
+    }
+    try:
+        return subprocess.run(
+            command,
+            cwd=workspace,
+            env=environment,
+            stdout=stdout if stdout is not None else subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            timeout=GIT_TIMEOUT_SECONDS,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired) as error:
+        raise RunnerProtocolError("The runner could not compare the files.") from error
+
+
+def require_git(result: subprocess.CompletedProcess[bytes]) -> None:
+    if result.returncode:
+        raise RunnerProtocolError("The runner could not compare the files.")
+
+
+def record_baseline(session_root: Path, workspace: Path) -> None:
+    """Commit the unpacked files, before the sandbox can change them."""
+    git_dir = session_root / BASELINE_DIR
+    git_dir.mkdir(mode=0o700)
+    require_git(
+        git(session_root, workspace, "init", "--quiet", "--bare", work_tree=False)
+    )
+    # Forced, so a committed file that matches .gitignore is compared too.
+    require_git(git(session_root, workspace, "add", "--all", "--force", "."))
+    require_git(
+        git(
+            session_root,
+            workspace,
+            "commit",
+            "--quiet",
+            "--no-verify",
+            "--allow-empty",
+            "--message=Run baseline",
+        )
+    )
+
+
+def collect(session_root: Path, workspace: Path) -> Path:
+    """Build the session's archive once. Later calls return the same one.
+
+    The sandbox must already be stopped, so nothing changes the files while
+    they are read.
+    """
+    archive_path = session_root / COLLECTION_FILE
+    if archive_path.is_file():
+        return archive_path
+    partial = session_root / f"{COLLECTION_FILE}.partial"
+    patch_path = session_root / CHANGES_NAME
+    # Sessions that did not ask for changes have no baseline to compare.
+    manifest: dict[str, Any] = {
+        "changes": compare(session_root, workspace, patch_path)
+        if (session_root / BASELINE_DIR).is_dir()
+        else NO_CHANGES
+    }
+    with tarfile.open(partial, mode="w:gz") as archive:
+        if manifest["changes"]["captured"]:
+            add_file(archive, patch_path, CHANGES_NAME)
+        manifest["outputs"], manifest["skipped"] = add_outputs(
+            archive, session_root / OUTPUTS_DIR
+        )
+        add_bytes(archive, MANIFEST_NAME, json.dumps(manifest).encode())
+    patch_path.unlink(missing_ok=True)
+    os.replace(partial, archive_path)
+    return archive_path
+
+
+def compare(session_root: Path, workspace: Path, patch_path: Path) -> dict[str, Any]:
+    """Write the changes since the baseline, if they fit, and describe them."""
+    # Not forced: new files that match .gitignore, such as build output, stay out.
+    require_git(git(session_root, workspace, "add", "--all", "."))
+    names = git(session_root, workspace, "diff", "--cached", "--name-only", "-z")
+    require_git(names)
+    paths = [
+        item.decode(errors="replace") for item in names.stdout.split(b"\0") if item
+    ]
+    with patch_path.open("wb") as stream:
+        require_git(
+            git(
+                session_root,
+                workspace,
+                "diff",
+                "--cached",
+                "--binary",
+                "--no-color",
+                "--no-ext-diff",
+                "--no-textconv",
+                "--full-index",
+                stdout=stream,
+            )
+        )
+    size = patch_path.stat().st_size
+    return {
+        "file_count": len(paths),
+        "files": paths[:MAX_LISTED_CHANGES],
+        "size_bytes": size,
+        "captured": bool(paths) and size <= MAX_CHANGES_BYTES,
+        "too_large": size > MAX_CHANGES_BYTES,
+    }
+
+
+def add_outputs(
+    archive: tarfile.TarFile, outputs: Path
+) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:
+    saved: list[dict[str, Any]] = []
+    skipped: list[dict[str, str]] = []
+    if not outputs.is_dir():
+        return saved, skipped
+    total = 0
+    for root, directories, files in os.walk(outputs, followlinks=False):
+        # Linked folders are never entered, and are listed as skipped.
+        for name in [item for item in directories if (Path(root) / item).is_symlink()]:
+            directories.remove(name)
+            relative = (Path(root) / name).relative_to(outputs).as_posix()
+            skipped.append({"name": relative, "reason": "Not a regular file."})
+        directories.sort()
+        for name in sorted(files):
+            path = Path(root) / name
+            relative = path.relative_to(outputs).as_posix()
+            reason = output_problem(path, relative, len(saved), total)
+            if reason:
+                skipped.append({"name": relative, "reason": reason})
+                continue
+            size, digest = add_file(archive, path, f"{OUTPUTS_DIR}/{relative}")
+            total += size
+            saved.append({"name": relative, "size_bytes": size, "sha256": digest})
+    return saved, skipped
+
+
+def output_problem(path: Path, relative: str, count: int, total: int) -> str:
+    info = path.lstat()
+    if not stat.S_ISREG(info.st_mode):
+        return "Not a regular file."
+    if not safe_output_name(relative):
+        return "The file name is not allowed."
+    if count >= MAX_OUTPUT_FILES:
+        return f"Only the first {MAX_OUTPUT_FILES} files are kept."
+    if total + info.st_size > MAX_OUTPUT_BYTES:
+        return "The files are larger than the limit."
+    return ""
+
+
+def safe_output_name(name: str) -> bool:
+    """A relative path of plain parts, safe to use as a storage path."""
+    path = PurePosixPath(name)
+    return (
+        0 < len(name) <= 255
+        and not path.is_absolute()
+        and all(part not in {"", ".", ".."} for part in path.parts)
+        and all(char.isprintable() and char != "\\" for char in name)
+    )
+
+
+def add_file(archive: tarfile.TarFile, path: Path, name: str) -> tuple[int, str]:
+    """Add one regular file under a fixed owner and mode. Links are refused."""
+    descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
+    with os.fdopen(descriptor, "rb") as stream:
+        info = os.fstat(stream.fileno())
+        if not stat.S_ISREG(info.st_mode):
+            raise RunnerProtocolError("A collected file is not a regular file.")
+        member = tarfile.TarInfo(name)
+        member.size = info.st_size
+        member.mode = 0o600
+        reader = HashingReader(stream)
+        archive.addfile(member, reader)
+        return info.st_size, reader.digest.hexdigest()
+
+
+class HashingReader:
+    """Passes a stream to tarfile and keeps the SHA-256 of what it read."""
+
+    def __init__(self, stream: IO[bytes]) -> None:
+        self.stream = stream
+        self.digest = hashlib.sha256()
+
+    def read(self, size: int = -1) -> bytes:
+        chunk = self.stream.read(size)
+        self.digest.update(chunk)
+        return chunk
+
+
+def add_bytes(archive: tarfile.TarFile, name: str, content: bytes) -> None:
+    member = tarfile.TarInfo(name)
+    member.size = len(content)
+    member.mode = 0o600
+    archive.addfile(member, io.BytesIO(content))

secondshift/runner/dao.py

@@ -0,0 +1,64 @@
+"""Queries and writes on runner sessions and the commands run in them."""
+
+from typing import Any
+
+from secondshift.runner.models import RunnerCommand, RunnerSession
+
+
+def has_sessions() -> bool:
+    """Whether any session exists. The health check uses it to reach the database."""
+    return RunnerSession.objects.exists()
+
+
+def get_session(session_id: str) -> RunnerSession:
+    """The session. Raises RunnerSession.DoesNotExist when there is none."""
+    return RunnerSession.objects.get(session_id=session_id)
+
+
+def lock_session(session_id: str, active_only: bool = False) -> RunnerSession | None:
+    """Lock the session for this transaction. None when it does not exist."""
+    sessions = RunnerSession.objects.select_for_update().filter(session_id=session_id)
+    if active_only:
+        sessions = sessions.filter(status=RunnerSession.Status.ACTIVE)
+    return sessions.first()
+
+
+def is_session_active(session_id: str) -> bool:
+    return RunnerSession.objects.filter(
+        session_id=session_id, status=RunnerSession.Status.ACTIVE
+    ).exists()
+
+
+def create_session(**fields: Any) -> RunnerSession:
+    """Record a new session. Raises IntegrityError when the run already has one."""
+    return RunnerSession.objects.create(**fields)
+
+
+def set_session_status(session_id: str, status: RunnerSession.Status) -> None:
+    RunnerSession.objects.filter(session_id=session_id).update(status=status)
+
+
+def session_command(session: RunnerSession, call_id: str) -> RunnerCommand | None:
+    """The command already reserved for this call, if any."""
+    return RunnerCommand.objects.filter(session=session, call_id=call_id).first()
+
+
+def create_command(
+    session: RunnerSession, call_id: str, fingerprint: str
+) -> RunnerCommand:
+    """Reserve a call before running it."""
+    return RunnerCommand.objects.create(
+        session=session, call_id=call_id, fingerprint=fingerprint
+    )
+
+
+def complete_command(
+    session: RunnerSession, call_id: str, exit_code: int, output: str, timed_out: bool
+) -> None:
+    """Store the outcome of a reserved call."""
+    RunnerCommand.objects.filter(session=session, call_id=call_id).update(
+        status=RunnerCommand.Status.COMPLETED,
+        exit_code=exit_code,
+        output=output,
+        timed_out=timed_out,
+    )

secondshift/runner/dev.py

@@ -0,0 +1,19 @@
+"""Start the development runner with the exact locally built agent image."""
+
+import os
+import sys
+
+from secondshift.config.constants import AppEnv
+from secondshift.runner.images import resolve_image_id
+
+
+def main() -> None:
+    image = os.environ.get(AppEnv.AGENT_IMAGE, "")
+    if not image:
+        raise ValueError("The development agent image must be configured.")
+    os.environ[AppEnv.AGENT_IMAGE] = resolve_image_id(image)
+    os.execv(sys.executable, [sys.executable, "-m", "gunicorn", "--reload"])
+
+
+if __name__ == "__main__":
+    main()

secondshift/runner/images.py

@@ -0,0 +1,57 @@
+"""Build the agent image from the files shipped in the package."""
+
+import re
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+from secondshift.config.constants import LOCAL_AGENT_TAG
+
+AGENT_IMAGE_DIR = Path(__file__).parent / "agent_image"
+BUILD_SECONDS = 1800
+
+
+def resolve_image_id(image: str) -> str:
+    """The immutable ID behind a local image name."""
+    result = subprocess.run(
+        ["docker", "image", "inspect", "--format", "{{.Id}}", image],
+        capture_output=True,
+        text=True,
+        check=True,
+        timeout=30,
+    )
+    image_id = result.stdout.strip()
+    if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
+        raise ValueError("Docker did not return an immutable agent image ID.")
+    return image_id
+
+
+def build_agent_image(tag: str = LOCAL_AGENT_TAG) -> str:
+    """Build the pinned agent image and return its immutable ID.
+
+    The build context holds only the agent files, laid out as in the checkout,
+    so Docker never uploads the rest of the installed packages.
+    """
+    with tempfile.TemporaryDirectory(prefix="secondshift-agent-") as context:
+        target = Path(context) / "secondshift" / "runner" / "agent_image"
+        shutil.copytree(
+            AGENT_IMAGE_DIR, target, ignore=shutil.ignore_patterns("__pycache__")
+        )
+        subprocess.run(
+            [
+                "docker",
+                "build",
+                "--quiet",
+                "-f",
+                str(target / "Dockerfile"),
+                "-t",
+                tag,
+                context,
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=BUILD_SECONDS,
+        )
+    return resolve_image_id(tag)

secondshift/runner/migrations/0001_initial.py

@@ -0,0 +1,48 @@
+# Generated by Django 6.0.9 on 2026-10-26 14:22
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RunnerSession',
+            fields=[
+                ('session_id', models.CharField(max_length=80, primary_key=True, serialize=False)),
+                ('run_id', models.PositiveBigIntegerField(db_index=True)),
+                ('repository', models.CharField(max_length=300)),
+                ('commit_sha', models.CharField(max_length=64)),
+                ('image', models.CharField(max_length=300)),
+                ('container_name', models.CharField(max_length=120, unique=True)),
+                ('workspace_path', models.TextField()),
+                ('status', models.CharField(choices=[('starting', 'Starting'), ('active', 'Active'), ('stopped', 'Stopped'), ('error', 'Error')], default='starting', max_length=16)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='RunnerCommand',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('call_id', models.CharField(max_length=200)),
+                ('fingerprint', models.CharField(max_length=64)),
+                ('status', models.CharField(choices=[('running', 'Running'), ('completed', 'Completed')], default='running', max_length=16)),
+                ('exit_code', models.IntegerField(blank=True, null=True)),
+                ('output', models.TextField(default='')),
+                ('timed_out', models.BooleanField(default=False)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='commands', to='runner.runnersession')),
+            ],
+            options={
+                'constraints': [models.UniqueConstraint(fields=('session', 'call_id'), name='runner_unique_session_call')],
+            },
+        ),
+    ]

secondshift/runner/migrations/0002_session_source_sha256.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.9 on 2026-10-29 22:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('runner', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='runnersession',
+            name='source_sha256',
+            field=models.CharField(blank=True, max_length=64),
+        ),
+    ]

secondshift/runner/migrations/0003_session_collect_changes.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.9 on 2026-11-03 20:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('runner', '0002_session_source_sha256'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='runnersession',
+            name='collect_changes',
+            field=models.BooleanField(default=False),
+        ),
+    ]

secondshift/runner/migrations/0004_session_history_sha256.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.9 on 2026-11-07 15:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('runner', '0003_session_collect_changes'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='runnersession',
+            name='history_sha256',
+            field=models.CharField(blank=True, max_length=64),
+        ),
+    ]

secondshift/runner/migrations/0005_require_source_sha256.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.9 on 2026-11-08 23:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('runner', '0004_session_history_sha256'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='runnersession',
+            name='source_sha256',
+            field=models.CharField(max_length=64),
+        ),
+    ]

secondshift/runner/migrations/0006_session_skill_sha256.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.9 on 2026-11-11 19:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('runner', '0005_require_source_sha256'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='runnersession',
+            name='skill_sha256',
+            field=models.CharField(blank=True, max_length=64),
+        ),
+    ]

secondshift/runner/models.py

@@ -0,0 +1,53 @@
+from django.db import models
+
+
+class RunnerSession(models.Model):
+    class Status(models.TextChoices):
+        STARTING = "starting", "Starting"
+        ACTIVE = "active", "Active"
+        STOPPED = "stopped", "Stopped"
+        ERROR = "error", "Error"
+
+    session_id = models.CharField(primary_key=True, max_length=80)
+    run_id = models.PositiveBigIntegerField(db_index=True)
+    repository = models.CharField(max_length=300)
+    commit_sha = models.CharField(max_length=64)
+    image = models.CharField(max_length=300)
+    source_sha256 = models.CharField(max_length=64)
+    collect_changes = models.BooleanField(default=False)
+    history_sha256 = models.CharField(max_length=64, blank=True)
+    skill_sha256 = models.CharField(max_length=64, blank=True)
+    container_name = models.CharField(max_length=120, unique=True)
+    workspace_path = models.TextField()
+    status = models.CharField(
+        max_length=16, choices=Status.choices, default=Status.STARTING
+    )
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+
+class RunnerCommand(models.Model):
+    class Status(models.TextChoices):
+        RUNNING = "running", "Running"
+        COMPLETED = "completed", "Completed"
+
+    session = models.ForeignKey(
+        RunnerSession, on_delete=models.CASCADE, related_name="commands"
+    )
+    call_id = models.CharField(max_length=200)
+    fingerprint = models.CharField(max_length=64)
+    status = models.CharField(
+        max_length=16, choices=Status.choices, default=Status.RUNNING
+    )
+    exit_code = models.IntegerField(null=True, blank=True)
+    output = models.TextField(default="")
+    timed_out = models.BooleanField(default=False)
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=("session", "call_id"), name="runner_unique_session_call"
+            )
+        ]