secondshift 0.1.0b12__py3-none-any.whl

secondshift/runner/sources.py

@@ -0,0 +1,130 @@
+"""Source snapshots the control plane uploads for a run to unpack.
+
+The runner keeps no repositories. Each snapshot is named by its SHA-256, kept
+only until its session starts, and swept after an hour if no session uses it.
+"""
+
+import hashlib
+import os
+import re
+import tarfile
+import tempfile
+import time
+from pathlib import Path
+from typing import IO
+
+from django.conf import settings
+
+from secondshift.config.constants import (
+    MAX_SOURCE_FILES,
+    MAX_SOURCE_SNAPSHOT_BYTES,
+    MAX_SOURCE_UNPACKED_BYTES,
+)
+from secondshift.runner.protocol import RunnerProtocolError
+
+SOURCE_SHA256 = re.compile(r"[0-9a-f]{64}")
+SOURCE_MAX_AGE_SECONDS = 60 * 60
+CHUNK_BYTES = 1024 * 1024
+
+
+def sources_root() -> Path:
+    return (settings.RUNNER_WORK_ROOT / "sources").resolve()
+
+
+def source_path(sha256: str) -> Path:
+    if not SOURCE_SHA256.fullmatch(sha256):
+        raise RunnerProtocolError("A source is named by its SHA-256.")
+    return sources_root() / f"{sha256}.tar.gz"
+
+
+def has_source(sha256: str) -> bool:
+    """Whether the snapshot is here. Asking keeps it for another hour."""
+    path = source_path(sha256)
+    try:
+        os.utime(path)
+    except FileNotFoundError:
+        return False
+    return True
+
+
+def receive_source(sha256: str, stream: IO[bytes], length: int | None) -> None:
+    """Store an uploaded snapshot if its bytes match its name."""
+    target = source_path(sha256)
+    if length is None:
+        raise RunnerProtocolError("The source upload needs a Content-Length.")
+    if length > MAX_SOURCE_SNAPSHOT_BYTES:
+        raise RunnerProtocolError("The source is too large.")
+    root = sources_root()
+    root.mkdir(parents=True, exist_ok=True, mode=0o700)
+    sweep_sources()
+    descriptor, name = tempfile.mkstemp(dir=root, prefix="upload-")
+    temporary = Path(name)
+    try:
+        digest = hashlib.sha256()
+        remaining = length
+        with os.fdopen(descriptor, "wb") as output:
+            while remaining:
+                chunk = stream.read(min(CHUNK_BYTES, remaining))
+                if not chunk:
+                    raise RunnerProtocolError("The source upload ended early.")
+                remaining -= len(chunk)
+                digest.update(chunk)
+                output.write(chunk)
+        if digest.hexdigest() != sha256:
+            raise RunnerProtocolError("The source does not match its SHA-256.")
+        os.replace(temporary, target)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def sweep_sources() -> None:
+    """Remove snapshots and partial uploads no session started with."""
+    root = sources_root()
+    if not root.is_dir():
+        return
+    cutoff = time.time() - SOURCE_MAX_AGE_SECONDS
+    for path in root.iterdir():
+        try:
+            if path.is_file() and path.stat().st_mtime < cutoff:
+                path.unlink()
+        except FileNotFoundError:
+            continue
+
+
+def unpack_source(sha256: str, workspace: Path) -> None:
+    """Unpack an uploaded snapshot into a session's empty workspace."""
+    path = source_path(sha256)
+    if not path.is_file():
+        raise RunnerProtocolError("The run's source files were not uploaded.")
+    if file_sha256(path) != sha256:
+        raise RunnerProtocolError("The run's source files do not match their SHA-256.")
+    try:
+        with tarfile.open(path, mode="r:gz") as archive:
+            check_members(archive)
+            archive.extractall(workspace, filter="data")
+    except (tarfile.TarError, OSError, EOFError) as error:
+        raise RunnerProtocolError(
+            "The runner could not unpack the run's source files."
+        ) from error
+
+
+def check_members(archive: tarfile.TarFile) -> None:
+    # Stop at the first entry past a limit, so a compression bomb is never
+    # read to the end.
+    size = 0
+    for files, member in enumerate(archive, start=1):
+        size += member.size
+        if files > MAX_SOURCE_FILES or size > MAX_SOURCE_UNPACKED_BYTES:
+            raise RunnerProtocolError("The run's source files are too large.")
+        if not (member.isreg() or member.isdir() or member.issym()):
+            raise RunnerProtocolError(
+                "The run's source files may only hold files, folders and links."
+            )
+
+
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        while chunk := stream.read(CHUNK_BYTES):
+            digest.update(chunk)
+    return digest.hexdigest()

secondshift/runner/urls.py

@@ -0,0 +1,7 @@
+"""The runner deployment exposes only its narrow machine-to-machine API."""
+
+from django.urls import path
+
+from secondshift.runner.api import api
+
+urlpatterns = [path("", api.urls)]

secondshift/runs/access.py

@@ -0,0 +1,13 @@
+from django.http import Http404, HttpRequest
+
+from secondshift.runs.dao import find_workspace_run
+from secondshift.runs.models import Run
+from secondshift.workspaces.access import current_workspace
+
+
+def run_for_request(request: HttpRequest, run_id: int) -> Run:
+    workspace = current_workspace(request)
+    run = find_workspace_run(workspace.pk if workspace else None, run_id)
+    if run is None:
+        raise Http404
+    return run

secondshift/runs/admin.py

@@ -0,0 +1,89 @@
+from django.contrib import admin
+from django.http import FileResponse, Http404, HttpRequest
+from django.urls import URLPattern, URLResolver, path, reverse
+from django.utils.html import format_html
+
+from secondshift.runs.artifacts import artifact_storage
+from secondshift.runs.dao import get_run
+from secondshift.runs.models import Run, RunEvent
+
+
+class RunEventInline(admin.TabularInline):
+    model = RunEvent
+    extra = 0
+    can_delete = False
+    readonly_fields = ("level", "message", "metadata", "created_at")
+
+
+@admin.register(Run)
+class RunAdmin(admin.ModelAdmin):
+    list_display = (
+        "display_name",
+        "status",
+        "scheduled_for",
+        "sandbox_runtime",
+        "finished_at",
+    )
+    list_filter = ("status", "sandbox_runtime")
+    search_fields = ("job__name", "job__repository__full_name", "commit_sha")
+    readonly_fields = (
+        "job",
+        "task",
+        "status",
+        "commit_sha",
+        "scheduled_for",
+        "started_at",
+        "finished_at",
+        "sandbox_runtime",
+        "exit_code",
+        "dbos_workflow_id",
+        "summary",
+        "input_snapshot",
+        "execution_plan",
+        "output_excerpt",
+        "full_output",
+        "output_size_bytes",
+        "output_truncated",
+        "output_sha256",
+        "created_at",
+    )
+    exclude = ("output_path",)
+    inlines = [RunEventInline]
+
+    def get_urls(self) -> list[URLPattern | URLResolver]:
+        return [
+            path(
+                "<int:run_id>/output/",
+                self.admin_site.admin_view(self.output_view),
+                name="runs_run_output",
+            ),
+            *super().get_urls(),
+        ]
+
+    @admin.display(description="Full output")
+    def full_output(self, run: Run) -> str:
+        if not run.output_path:
+            return "-"
+        url = reverse("admin:runs_run_output", args=[run.pk])
+        return format_html('<a href="{}">Download output.log</a>', url)
+
+    def output_view(self, request: HttpRequest, run_id: int) -> FileResponse:
+        try:
+            run = get_run(run_id)
+        except Run.DoesNotExist:
+            raise Http404 from None
+        if not self.has_view_permission(request, run) or not run.output_path:
+            raise Http404
+        return FileResponse(
+            artifact_storage().open(run.output_path, "rb"),
+            as_attachment=True,
+            filename=f"run-{run.pk}-output.log",
+            content_type="text/plain; charset=utf-8",
+        )
+
+
+@admin.register(RunEvent)
+class RunEventAdmin(admin.ModelAdmin):
+    list_display = ("run", "level", "message", "created_at")
+    list_filter = ("level",)
+    readonly_fields = ("run", "level", "message", "metadata", "created_at")

secondshift/runs/api.py

@@ -0,0 +1,211 @@
+from pathlib import PurePosixPath
+
+from django.http import FileResponse, Http404, HttpRequest
+from ninja import Query, Router, Status
+from ninja.errors import HttpError
+from ninja.security import django_auth
+
+from secondshift.config.constants import RunStart
+from secondshift.jobs.access import job_for_request
+from secondshift.repositories.access import repository_for_request
+from secondshift.runs.access import run_for_request
+from secondshift.runs.artifacts import artifact_storage
+from secondshift.runs.deliveries import DeliveryQueueUnavailable, retry_delivery
+from secondshift.runs.models import RunDelivery
+from secondshift.runs.queue import cancel_queued_task
+from secondshift.runs.reporting.queries import RunFilters, run_page
+from secondshift.runs.schemas import (
+    RunAgainSchema,
+    RunDeliverySchema,
+    RunDetailSchema,
+    RunFileName,
+    RunInProgressSchema,
+    RunListQuery,
+    RunPageSchema,
+    RunSchema,
+    TaskSubmissionSchema,
+)
+from secondshift.runs.serialization import (
+    run_file_path,
+    saved_file_path,
+    serialize_delivery,
+    serialize_run,
+    serialize_run_detail,
+    serialize_submission,
+)
+from secondshift.runs.services import RunInProgress, RunRefused, start_manual_run
+from secondshift.runs.submissions import SkillSubmission, resubmission, submit_skill
+from secondshift.workspaces.access import (
+    Action,
+    current_workspace,
+    require,
+)
+
+router = Router(auth=django_auth, tags=["runs"])
+
+
+@router.post("/runs/{run_id}/cancel", response=RunSchema)
+def cancel_task(request: HttpRequest, run_id: int) -> RunSchema:
+    require(request, Action.RUN_JOBS)
+    run = run_for_request(request, run_id)
+    try:
+        return serialize_run(cancel_queued_task(run))
+    except RunRefused as error:
+        raise HttpError(409, str(error)) from error
+
+
+@router.post("/runs/{run_id}/again", response=TaskSubmissionSchema)
+def run_task_again(
+    request: HttpRequest, run_id: int, payload: RunAgainSchema
+) -> TaskSubmissionSchema:
+    """Submit a task run's task again, as it was first submitted."""
+    require(request, Action.RUN_JOBS)
+    run = run_for_request(request, run_id)
+    try:
+        submission = resubmission(run, payload.idempotency_key)
+        result = submit_skill(run.repository, submission, request.user, RunStart.MANUAL)
+    except RunRefused as error:
+        raise HttpError(409, str(error)) from error
+    return serialize_submission(result.run, result.started)
+
+
+@router.post(
+    "/runs/{run_id}/deliveries/{delivery_id}/retry", response=RunDeliverySchema
+)
+def retry_run_delivery(
+    request: HttpRequest, run_id: int, delivery_id: int
+) -> RunDeliverySchema:
+    require(request, Action.RUN_JOBS)
+    run = run_for_request(request, run_id)
+    try:
+        return serialize_delivery(retry_delivery(run, delivery_id))
+    except RunDelivery.DoesNotExist as error:
+        raise Http404 from error
+    except RunRefused as error:
+        raise HttpError(409, str(error)) from error
+    except DeliveryQueueUnavailable as error:
+        raise HttpError(
+            503, "The execution queue is unavailable. Try again shortly."
+        ) from error
+
+
+@router.get("/runs", response=RunPageSchema)
+def runs(request: HttpRequest, query: Query[RunListQuery]) -> RunPageSchema:
+    workspace = current_workspace(request)
+    if not workspace:
+        return RunPageSchema(
+            items=[], count=0, page=1, page_size=query.page_size, status_counts={}
+        )
+    result = run_page(
+        workspace,
+        RunFilters(
+            q=query.q,
+            statuses=[str(status) for status in query.status],
+            job_id=query.job_id,
+            repository_id=query.repository_id,
+            period=query.period,
+            sort=query.sort,
+        ),
+        query.page,
+        query.page_size,
+    )
+    return RunPageSchema(
+        items=[serialize_run(run) for run in result.runs],
+        count=result.count,
+        page=result.page,
+        page_size=result.page_size,
+        status_counts=result.status_counts,
+    )
+
+
+@router.get("/runs/{run_id}", response=RunDetailSchema)
+def run_detail(request: HttpRequest, run_id: int) -> RunDetailSchema:
+    return serialize_run_detail(run_for_request(request, run_id))
+
+
+def stored_file_response(
+    path: str, filename: str, download: bool, content_type: str
+) -> FileResponse:
+    response = FileResponse(
+        artifact_storage().open(path, "rb"),
+        as_attachment=download,
+        filename=filename,
+        content_type=content_type,
+    )
+    response["X-Content-Type-Options"] = "nosniff"
+    response["Cache-Control"] = "private, no-store"
+    return response
+
+
+@router.get("/runs/{run_id}/files/{name}")
+def run_file(
+    request: HttpRequest, run_id: int, name: RunFileName, download: bool = False
+) -> FileResponse:
+    """Stream a stored run file through the configured storage backend.
+
+    Always served as plain text, so a browser never renders repository or model
+    content as HTML; the dashboard renders Markdown itself, sanitized.
+    """
+    run = run_for_request(request, run_id)
+    path = run_file_path(run, name)
+    if not path:
+        raise Http404
+    return stored_file_response(
+        path, f"run-{run.pk}-{name}.md", download, "text/plain; charset=utf-8"
+    )
+
+
+@router.get("/runs/{run_id}/changes")
+def run_changes_patch(
+    request: HttpRequest, run_id: int, download: bool = False
+) -> FileResponse:
+    """The agent's file changes as a patch, always served as plain text."""
+    run = run_for_request(request, run_id)
+    path = run.collected.changes.path if run.collected else ""
+    if not path:
+        raise Http404
+    return stored_file_response(
+        path, f"run-{run.pk}-changes.patch", download, "text/plain; charset=utf-8"
+    )
+
+
+@router.get("/runs/{run_id}/saved-files/{path:name}")
+def run_saved_file(request: HttpRequest, run_id: int, name: str) -> FileResponse:
+    """One file the agent saved. Always a download, never rendered.
+
+    The agent chose its name and content, so the browser must not open it as
+    a page on this site.
+    """
+    run = run_for_request(request, run_id)
+    path = saved_file_path(run, name)
+    if not path:
+        raise Http404
+    return stored_file_response(
+        path, PurePosixPath(name).name, True, "application/octet-stream"
+    )
+
+
+@router.post("/jobs/{job_id}/run", response={200: RunSchema, 409: RunInProgressSchema})
+def run_job(
+    request: HttpRequest, job_id: int
+) -> RunSchema | Status[RunInProgressSchema]:
+    job = job_for_request(request, job_id)
+    require(request, Action.RUN_JOBS)
+    try:
+        run = start_manual_run(job, submitted_by=request.user)
+    except RunInProgress as error:
+        return Status(409, RunInProgressSchema(detail=str(error), run_id=error.run.pk))
+    except RunRefused as error:
+        raise HttpError(409, str(error)) from error
+    return serialize_run(run)
+
+
+@router.post("/tasks", response=TaskSubmissionSchema)
+def submit_task(request: HttpRequest, payload: SkillSubmission) -> TaskSubmissionSchema:
+    require(request, Action.RUN_JOBS)
+    repository = repository_for_request(request, payload.repository_id)
+    try:
+        result = submit_skill(repository, payload, request.user, RunStart.MANUAL)
+    except RunRefused as error:
+        raise HttpError(409, str(error)) from error
+    return serialize_submission(result.run, result.started)

secondshift/runs/apps.py

@@ -0,0 +1,11 @@
+from importlib import import_module
+
+from django.apps import AppConfig
+
+
+class RunsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "secondshift.runs"
+
+    def ready(self) -> None:
+        import_module("secondshift.runs.signals")