secondshift 0.1.0b12__py3-none-any.whl

secondshift/config/settings/runner.py

@@ -0,0 +1,140 @@
+"""Settings for the separately deployed command runner.
+
+This module deliberately does not import the control-plane settings. The runner
+has its own apps, database credentials, authentication token, and network policy.
+"""
+
+import os
+import secrets
+from pathlib import Path
+
+from django.core.exceptions import ImproperlyConfigured
+from environ import Env
+
+from secondshift.config.constants import (
+    DEFAULT_SANDBOX_IMAGE,
+    ENVIRONMENT_VAR,
+    LOCAL_AGENT_TAG,
+    LOCAL_GATEWAY_URL,
+    AppEnv,
+    Environment,
+    SandboxRuntime,
+    Service,
+    local_agent_broker_url,
+)
+from secondshift.config.logging import logging_config
+
+env = Env()
+# On a single host the runner shares the data folder, but reads only runner.env
+# (its token), never the control plane's .env with the encryption key.
+DATA_DIR = Path(env.str(AppEnv.DATA_DIR, default="~/.secondshift")).expanduser()
+Env.read_env(DATA_DIR / "runner.env")
+
+# Nothing in the runner signs data (no sessions, no auth cookies), so a
+# per-process key is enough and there is nothing to configure.
+SECRET_KEY = secrets.token_urlsafe(50)
+DEBUG = False
+ALLOWED_HOSTS = env.list(
+    AppEnv.RUNNER_ALLOWED_HOSTS, default=["127.0.0.1", "localhost"]
+)
+
+INSTALLED_APPS = ["django.contrib.contenttypes", "secondshift.runner"]
+MIDDLEWARE = ["django.middleware.security.SecurityMiddleware"]
+ROOT_URLCONF = "secondshift.runner.urls"
+
+USE_TZ = True
+TIME_ZONE = "UTC"
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+DATA_UPLOAD_MAX_MEMORY_SIZE = 64 * 1024
+
+RUNNER_WORK_ROOT = Path(
+    env.str(AppEnv.RUNNER_WORK_ROOT, default=str(DATA_DIR / "runner"))
+).resolve()
+RUNNER_WORK_ROOT.mkdir(parents=True, exist_ok=True, mode=0o700)
+RUNNER_DATABASE_URL = env.str(
+    AppEnv.RUNNER_DATABASE_URL,
+    default=f"sqlite:///{RUNNER_WORK_ROOT / 'runner.sqlite3'}",
+)
+DATABASES = {
+    "default": {
+        **env.db_url_config(RUNNER_DATABASE_URL),
+        "CONN_MAX_AGE": env.int(AppEnv.RUNNER_DB_CONN_MAX_AGE_SECONDS, default=60),
+        "CONN_HEALTH_CHECKS": True,
+    }
+}
+
+RUNNER_SERVICE_TOKEN = env.str(AppEnv.RUNNER_SERVICE_TOKEN, default="")
+RUNNER_IMAGE = env.str(AppEnv.RUNNER_IMAGE, default=DEFAULT_SANDBOX_IMAGE)
+RUNNER_RUNTIME = env.str(AppEnv.RUNNER_RUNTIME, default=SandboxRuntime.GVISOR)
+RUNNER_COMMAND_TIMEOUT_SECONDS = env.int(
+    "SECONDSHIFT_RUNNER_COMMAND_TIMEOUT_SECONDS", default=120
+)
+RUNNER_COMMAND_OUTPUT_MAX_BYTES = env.int(
+    "SECONDSHIFT_RUNNER_COMMAND_OUTPUT_MAX_BYTES", default=1024 * 1024
+)
+RUNNER_MEMORY = env.str("SECONDSHIFT_RUNNER_MEMORY", default="2g")
+RUNNER_CPUS = env.str("SECONDSHIFT_RUNNER_CPUS", default="2")
+RUNNER_PIDS_LIMIT = env.int("SECONDSHIFT_RUNNER_PIDS_LIMIT", default=256)
+# A runner started as root still runs sandboxes as an unprivileged user.
+UNPRIVILEGED_ID = 10001
+RUNNER_SANDBOX_UID = env.int(
+    "SECONDSHIFT_RUNNER_SANDBOX_UID", default=os.getuid() or UNPRIVILEGED_ID
+)
+RUNNER_SANDBOX_GID = env.int(
+    "SECONDSHIFT_RUNNER_SANDBOX_GID", default=os.getgid() or UNPRIVILEGED_ID
+)
+
+if len(RUNNER_SERVICE_TOKEN) < 32:
+    raise ImproperlyConfigured(
+        "SECONDSHIFT_RUNNER_SERVICE_TOKEN must contain at least 32 characters. "
+        "On a single host, start SecondShift once to generate it in runner.env."
+    )
+if RUNNER_RUNTIME not in set(SandboxRuntime):
+    raise ImproperlyConfigured(
+        f"SECONDSHIFT_RUNNER_RUNTIME must be one of {', '.join(SandboxRuntime)}"
+    )
+if RUNNER_COMMAND_TIMEOUT_SECONDS < 1:
+    raise ImproperlyConfigured("Runner command timeout must be positive")
+if RUNNER_COMMAND_OUTPUT_MAX_BYTES < 1024:
+    raise ImproperlyConfigured("Runner output limit must be at least 1024 bytes")
+if RUNNER_PIDS_LIMIT < 1:
+    raise ImproperlyConfigured("Runner PID limit must be positive")
+if RUNNER_SANDBOX_UID < 0 or RUNNER_SANDBOX_GID < 0:
+    raise ImproperlyConfigured("Runner sandbox UID and GID cannot be negative")
+# Each workspace is chowned to the sandbox user. Without root that only works
+# for the runner's own user and groups, so fail at startup, not on the first run.
+if os.geteuid() != 0 and (
+    os.geteuid() != RUNNER_SANDBOX_UID
+    or RUNNER_SANDBOX_GID not in {os.getegid(), *os.getgroups()}
+):
+    raise ImproperlyConfigured(
+        "SECONDSHIFT_RUNNER_SANDBOX_UID/GID name another user, which needs a "
+        "runner running as root. Unset them to use the runner's own user."
+    )
+
+LOGGING = logging_config(
+    env.str(AppEnv.LOG_LEVEL, default="INFO").upper(), Service.RUNNER
+)
+
+
+# CLI agent execution. Servers turn it on by setting an image pinned by digest.
+# `secondshift up` builds the local tag on one person's machine. Only there may
+# the runner resolve a tag, once per run. Servers must pin a digest.
+LOCAL = os.environ.get(ENVIRONMENT_VAR) == Environment.LOCAL
+AGENT_IMAGE = env.str(AppEnv.AGENT_IMAGE, default=LOCAL_AGENT_TAG if LOCAL else "")
+AGENT_IMAGE_TAGS_ALLOWED = LOCAL
+AGENT_RESOLV_CONF = env.str(AppEnv.AGENT_RESOLV_CONF, default="")
+AGENT_NETWORK = env.str(AppEnv.AGENT_NETWORK, default="bridge")
+AGENT_BROKER_URL = env.str(
+    AppEnv.AGENT_BROKER_URL,
+    default=local_agent_broker_url(
+        env.str(AppEnv.MODEL_GATEWAY_URL, default=LOCAL_GATEWAY_URL)
+    )
+    if LOCAL
+    else "",
+)
+
+AGENT_AUTH_ROOT = RUNNER_WORK_ROOT / "agent-auth"
+
+if AGENT_IMAGE:
+    DATA_UPLOAD_MAX_MEMORY_SIZE = 512 * 1024

secondshift/config/settings/test.py

@@ -0,0 +1,47 @@
+# ruff: noqa: F403, F405
+import os
+import tempfile
+
+from secondshift.config.constants import ENVIRONMENT_VAR, Environment
+
+# Tests use production defaults, whatever SECONDSHIFT_ENV the shell exports.
+os.environ[ENVIRONMENT_VAR] = Environment.TEST
+# Files go to a throwaway folder, and ~/.secondshift/.env is never read.
+os.environ["SECONDSHIFT_DATA_DIR"] = tempfile.mkdtemp(prefix="secondshift-test-")
+
+from secondshift.config.settings.base import *
+
+INSTALLED_APPS = [*INSTALLED_APPS, "secondshift.runner"]
+SECRET_KEY = "test-secret-key"
+FIELD_ENCRYPTION_KEY = "x2Ce3x8qg2ZDiZ0T3W2WXZkCw5rN0p8D2kMlcV1Wq3E="
+DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}}
+DBOS_SYSTEM_DATABASE_URL = "sqlite:///:memory:"
+PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
+SIGNUP_ENABLED = False
+# Tests run without collectstatic, so they cannot use the manifest storage.
+# An empty folder keeps WhiteNoise from warning that STATIC_ROOT is missing,
+# and keeps a local collectstatic build out of the tests.
+STATIC_ROOT = tempfile.mkdtemp(prefix="secondshift-static-")
+STORAGES = {
+    **STORAGES,
+    "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"},
+}
+
+# Tests install every first-party app in one Django process. Deployment settings
+# remain separate and are checked independently.
+RUNNER_SERVICE_TOKEN = "test-runner-service-token-value"
+RUNNER_WORK_ROOT = BASE_DIR / ".secondshift-runner-test"
+RUNNER_COMMAND_TIMEOUT_SECONDS = 120
+RUNNER_COMMAND_OUTPUT_MAX_BYTES = 1024 * 1024
+RUNNER_MEMORY = "2g"
+RUNNER_CPUS = "2"
+RUNNER_PIDS_LIMIT = 256
+RUNNER_SANDBOX_UID = os.getuid()
+RUNNER_SANDBOX_GID = os.getgid()
+AGENT_IMAGE_TAGS_ALLOWED = False
+
+# Internal services in tests have explicit addresses, independent of the shell.
+RUNNER_SERVICE_URL = "http://127.0.0.1:8002"
+MODEL_GATEWAY_URL = "http://127.0.0.1:8001"
+
+AGENT_RESOLV_CONF = ""

secondshift/config/static_storage.py

@@ -0,0 +1,45 @@
+"""Preserve Vite's asset names; Django hashes its own admin/account assets."""
+
+import re
+from collections.abc import Iterator
+from typing import Any
+
+from django.core.files.storage import Storage
+from whitenoise.storage import CompressedManifestStaticFilesStorage
+
+VITE_ASSET_PREFIX = "assets/"
+
+
+class StaticFilesStorage(CompressedManifestStaticFilesStorage):
+    def post_process(
+        self,
+        paths: dict[str, tuple[Storage, str]],
+        **options: Any,
+    ) -> Iterator[tuple[str, str, bool | Exception]]:
+        frontend = {
+            name: source
+            for name, source in paths.items()
+            if name.startswith(VITE_ASSET_PREFIX)
+        }
+        django = {
+            name: source for name, source in paths.items() if name not in frontend
+        }
+        yield from super().post_process(django, **options)
+        if options.get("dry_run"):
+            return
+        # Vite already hashes filenames and rewrites URLs in its HTML/CSS/JS.
+        # Register identity mappings without rewriting or duplicating those files.
+        self.hashed_files.update({name: name for name in frontend})
+        self.save_manifest()
+        for name in frontend:
+            yield name, name, False
+        for name, compressed in self.compress_files(frontend):
+            yield name, compressed, True
+
+
+def immutable_static_file(path: str, url: str) -> bool:
+    """Recognize Django's hashes and Vite's dash-separated content hashes."""
+    return bool(
+        re.search(r"\.[a-f0-9]{12}\.[^/]+$", url)
+        or re.search(r"/assets/[^/]+-[A-Za-z0-9_-]{8,}\.[^/]+$", url)
+    )

secondshift/config/urls.py

@@ -0,0 +1,42 @@
+from django.conf import settings
+from django.contrib import admin
+from django.http import FileResponse, Http404, HttpRequest
+from django.urls import include, path
+from health_check.views import HealthCheckView
+
+from secondshift.config.api import api
+from secondshift.core.signin import sign_in
+from secondshift.core.views import password_reset
+from secondshift.mcp_server.api import api as mcp_api
+from secondshift.repositories.github_views import github_callback, github_start
+from secondshift.workspaces.views import open_invite
+
+
+def spa(request: HttpRequest, path: str = "") -> FileResponse:
+    index = settings.WEB_DIR / "index.html"
+    if not index.exists():
+        raise Http404(
+            "Dashboard has not been built. Run npm --prefix dashboard run build."
+        )
+    return FileResponse(index.open("rb"), content_type="text/html")
+
+
+urlpatterns = [
+    path("github/connect/", github_start),
+    path("github/callback/", github_callback, name="github-callback"),
+    path("-/admin/", admin.site.urls),
+    path("accounts/password/reset/", password_reset),
+    path("accounts/", include("allauth.urls")),
+    path("sign-in/<str:token>/", sign_in, name="sign-in"),
+    path("invite/<str:token>/", open_invite, name="invite"),
+    path("api/", api.urls),
+    path("mcp", mcp_api.urls),
+    path(
+        "-/health/",
+        HealthCheckView.as_view(
+            checks=["health_check.checks.Database", "health_check.checks.Cache"]
+        ),
+    ),
+    path("", spa),
+    path("<path:path>", spa),
+]

secondshift/connections/access.py

@@ -0,0 +1,27 @@
+"""What a job could reach through an external connection."""
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+
+class AccessReason(StrEnum):
+    """Why a job cannot use an external connection. Stored in run plans."""
+
+    MCP_PROVIDER_UNSUPPORTED = "mcp_provider_unsupported"
+    MCP_SCOPE_UNKNOWN = "mcp_scope_unknown"
+    MCP_SERVER_UNKNOWN = "mcp_server_unknown"
+    GITHUB_REPOSITORY_UNKNOWN = "github_repository_unknown"
+    SENTRY_ORGANIZATION_UNKNOWN = "sentry_organization_unknown"
+    MCP_SOURCE_MISSING = "mcp_source_missing"
+    API_BASE_URL_UNKNOWN = "api_base_url_unknown"
+
+
+@dataclass(frozen=True)
+class Access:
+    """What a job could reach through a connection, in plain words."""
+
+    ok: bool
+    scope: str | None
+    text: str
+    # Set when the job cannot use the connection.
+    reason: AccessReason | None = None

secondshift/connections/admin.py

@@ -0,0 +1,25 @@
+from django.contrib import admin
+
+from secondshift.connections.models import Connection
+
+
+@admin.register(Connection)
+class ConnectionAdmin(admin.ModelAdmin):
+    list_display = (
+        "provider",
+        "display_name",
+        "binding_key",
+        "kind",
+        "workspace",
+        "status",
+        "expires_at",
+    )
+    list_filter = ("provider", "kind", "status", "auth_strategy")
+    search_fields = (
+        "display_name",
+        "binding_key",
+        "workspace__name",
+        "external_account_id",
+    )
+    exclude = ("access_token", "refresh_token", "delivery_url")
+    readonly_fields = ("created_at", "updated_at")

secondshift/connections/agents.py

@@ -0,0 +1,224 @@
+"""Person-owned CLI logins; only references enter job plans."""
+
+import json
+import logging
+import uuid
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.db import transaction
+from django.utils import timezone
+from pydantic import ValidationError
+
+from secondshift.config.constants import AgentBackend
+from secondshift.connections.dao import (
+    count_cleanup_attempt,
+    pending_cleanups,
+    queue_cleanup,
+)
+from secondshift.connections.models import AgentConnection, AgentCredentialCleanup
+from secondshift.runner.agent_image.outcomes import AgentOutcome
+from secondshift.runner.agent_protocol import AgentCheckInput
+from secondshift.runner.client import get_runner_client
+from secondshift.runner.protocol import RunnerProtocolError
+from secondshift.workspaces.dao import is_owner
+from secondshift.workspaces.models import Workspace
+
+logger = logging.getLogger(__name__)
+
+
+class AgentConnectionError(ValueError):
+    pass
+
+
+def validate_agent_auth(backend: str, value: str) -> str:
+    if backend == AgentBackend.CLAUDE_CODE:
+        if not value.startswith("sk-ant-oat") or any(c.isspace() for c in value):
+            raise AgentConnectionError(
+                "Use a Claude Code subscription token from claude setup-token."
+            )
+        return value
+    if backend != AgentBackend.CODEX:
+        raise AgentConnectionError("Choose Claude Code or Codex.")
+    try:
+        auth = json.loads(value)
+        tokens = auth["tokens"]
+        if not isinstance(tokens, dict) or not all(
+            isinstance(tokens.get(key), str) and tokens[key]
+            for key in ("access_token", "refresh_token", "id_token", "account_id")
+        ):
+            raise ValueError
+    except (ValueError, KeyError, TypeError):
+        raise AgentConnectionError(
+            "Use the auth.json from a Codex ChatGPT login."
+        ) from None
+    # Never import configuration, endpoints, API keys, hooks, or MCP servers.
+    return json.dumps(
+        {
+            "tokens": {
+                key: tokens[key]
+                for key in ("access_token", "refresh_token", "id_token", "account_id")
+            },
+            "last_refresh": auth.get("last_refresh"),
+        }
+    )
+
+
+def save_agent_connection(
+    workspace: Workspace,
+    owner: User,
+    backend: str,
+    display_name: str,
+    auth: str,
+    connection: AgentConnection | None = None,
+) -> AgentConnection:
+    if not settings.AGENT_IMAGE or not settings.AGENT_BROKER_URL:
+        raise AgentConnectionError("CLI execution is not configured on this server.")
+    cleaned = validate_agent_auth(backend, auth.strip())
+    if connection is None:
+        connection = AgentConnection(workspace=workspace, owner=owner, backend=backend)
+    if connection.workspace_id != workspace.pk or connection.owner_id != owner.pk:
+        raise AgentConnectionError("Only the login's owner can reconnect it.")
+    if connection.backend != backend:
+        raise AgentConnectionError("The agent backend cannot change during reconnect.")
+    connection.display_name = display_name.strip() or product_name(backend)
+    connection.auth_data = cleaned
+    connection.credential_key = uuid.uuid4()
+    connection.connected = True
+    # A check described the old login.
+    connection.last_check = None
+    connection.full_clean()
+    connection.save()
+    return connection
+
+
+def reconnect_agent_connection(
+    connection: AgentConnection,
+    owner: User,
+    backend: str,
+    display_name: str,
+    auth: str,
+) -> AgentConnection:
+    validate_agent_auth(backend, auth.strip())
+    if backend != connection.backend:
+        raise AgentConnectionError("The agent backend cannot change during reconnect.")
+    with transaction.atomic():
+        # Saving the new login gives it a new key, so runs stop using the old one.
+        cleanup = queue_cleanup(connection)
+        connection = save_agent_connection(
+            connection.workspace, owner, backend, display_name, auth, connection
+        )
+    remove_cached_credentials(cleanup)
+    return connection
+
+
+def disconnect_agent_connection(connection: AgentConnection) -> bool:
+    """True when the runner's cached copy is gone too.
+
+    False means the runner did not answer. The worker removes the copy later.
+    """
+    with transaction.atomic():
+        connection.connected = False
+        connection.auth_data = ""
+        connection.save(update_fields=["connected", "auth_data", "updated_at"])
+        cleanup = queue_cleanup(connection)
+    return remove_cached_credentials(cleanup)
+
+
+def remove_cached_credentials(cleanup: AgentCredentialCleanup) -> bool:
+    """Ask the runner to remove one cached login. True once it is gone."""
+    try:
+        response = get_runner_client().send(
+            "DELETE", f"/v1/agent-credentials/{cleanup.credential_key.hex}"
+        )
+        removed = response.status_code == 204
+    except RunnerProtocolError:
+        removed = False
+    if removed:
+        cleanup.delete()
+        return True
+    count_cleanup_attempt(cleanup)
+    logger.warning("the runner kept agent login %s; will retry", cleanup.pk)
+    return False
+
+
+def retry_credential_cleanups() -> int:
+    """Remove the cached logins the runner kept earlier. Returns how many."""
+    return sum(remove_cached_credentials(cleanup) for cleanup in pending_cleanups(100))
+
+
+# What a check's outcome means for the login. Only a finished check and a
+# used-up allowance prove the sign-in works.
+CHECK_RESULTS: dict[str, tuple[bool, str]] = {
+    AgentOutcome.COMPLETED: (True, "Signed in. {product} answered a test message."),
+    AgentOutcome.QUOTA_EXHAUSTED: (
+        True,
+        "Signed in, but the plan's usage allowance is used up for now.",
+    ),
+    AgentOutcome.RECONNECT_REQUIRED: (
+        False,
+        "{product} refused the sign-in. Get a new login and reconnect.",
+    ),
+    AgentOutcome.TIMED_OUT: (
+        False,
+        "{product} did not answer in time. Check again later.",
+    ),
+}
+CHECK_FAILED = (False, "{product} could not finish the check. Check again later.")
+
+
+def product_name(backend: str) -> str:
+    return AgentBackend(backend).name.replace("_", " ").title()
+
+
+def check_agent_connection(connection: AgentConnection) -> AgentConnection:
+    """Send the login one fixed test message through the runner and keep the result.
+
+    It uses a little of the plan's allowance. The runner returns only a reason,
+    never the CLI's text, so nothing from the provider is stored.
+    """
+    if not connection.connected:
+        raise AgentConnectionError("Reconnect this login before checking it.")
+    client = get_runner_client()
+    try:
+        check_input = AgentCheckInput(
+            backend=connection.backend,
+            credential_key=connection.credential_key.hex,
+            auth=client.encrypted_auth(connection.auth_data),
+        )
+    except ValidationError:
+        raise AgentConnectionError(
+            "This login is larger than the runner accepts. Sign in again."
+        ) from None
+    try:
+        result = client.check_agent(check_input)
+    except RunnerProtocolError:
+        raise AgentConnectionError(
+            "The runner could not check the login now. It may be running a job "
+            "with it. Try again in a moment."
+        ) from None
+    reason = result.reason
+    ok, message = CHECK_RESULTS.get(reason, CHECK_FAILED)
+    connection.last_check = {
+        "ok": ok,
+        "reason": reason if reason in CHECK_RESULTS else AgentOutcome.AGENT_FAILED,
+        "message": message.format(product=product_name(connection.backend)),
+        "checked_at": timezone.now().isoformat(),
+    }
+    connection.save(update_fields=["last_check", "updated_at"])
+    return connection
+
+
+def sign_in_refused(connection: AgentConnection) -> bool:
+    """The last check proved the login no longer works."""
+    reason = (connection.last_check or {}).get("reason")
+    return reason == AgentOutcome.RECONNECT_REQUIRED
+
+
+def usable_agent_connection(connection: AgentConnection, workspace_id: int) -> bool:
+    return bool(
+        connection.connected
+        and connection.workspace_id == workspace_id
+        and connection.backend in (AgentBackend.CLAUDE_CODE, AgentBackend.CODEX)
+        and is_owner(workspace_id, connection.owner_id)
+    )