secondshift 0.1.0b12__py3-none-any.whl

secondshift/repositories/mirrors.py

@@ -0,0 +1,50 @@
+"""The control plane's Git mirror of each GitHub repository.
+
+A mirror is a cache. It is keyed by repository ID, so two workspaces never
+share one, and deleting it only costs a full fetch the next time.
+"""
+
+import fcntl
+import subprocess
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+from django.conf import settings
+
+# The commit the repository last synced. It keeps that commit's objects in the
+# mirror. Older commits are kept only until Git's own cleanup removes them.
+SYNCED_REF = "refs/secondshift/synced"
+
+
+def mirror_path(repository_id: int) -> Path:
+    return settings.SOURCE_CACHE_ROOT / "mirrors" / f"{repository_id}.git"
+
+
+@contextmanager
+def locked_mirror(repository_id: int) -> Iterator[Path]:
+    """The repository's bare mirror, created if missing, held under a lock."""
+    path = mirror_path(repository_id)
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    locks = settings.SOURCE_CACHE_ROOT / "locks"
+    locks.mkdir(parents=True, exist_ok=True, mode=0o700)
+    with (locks / f"{repository_id}.lock").open("a+b") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        if not path.exists():
+            subprocess.run(
+                ["git", "init", "--quiet", "--bare", str(path)],
+                check=True,
+                capture_output=True,
+                timeout=30,
+            )
+        yield path
+
+
+def has_commit(mirror: Path, commit_sha: str) -> bool:
+    completed = subprocess.run(
+        ["git", "-C", str(mirror), "cat-file", "-e", f"{commit_sha}^{{commit}}"],
+        capture_output=True,
+        check=False,
+        timeout=30,
+    )
+    return completed.returncode == 0

secondshift/repositories/models.py

@@ -0,0 +1,104 @@
+from pathlib import Path
+
+from django.core.exceptions import ValidationError
+from django.db import models
+
+from secondshift.connections.models import Connection
+from secondshift.workspaces.models import Workspace
+
+SYNC_ERROR_MAX_CHARS = 500
+
+
+class Repository(models.Model):
+    class SourceType(models.TextChoices):
+        GITHUB = "github", "GitHub"
+        LOCAL = "local", "Local"
+
+    workspace = models.ForeignKey(
+        Workspace,
+        on_delete=models.CASCADE,
+        related_name="repositories",
+    )
+    source_type = models.CharField(
+        max_length=16,
+        choices=SourceType.choices,
+        default=SourceType.GITHUB,
+    )
+    local_path = models.CharField(max_length=1000, blank=True)
+    github_connection = models.ForeignKey(
+        Connection,
+        on_delete=models.PROTECT,
+        related_name="repositories",
+        null=True,
+        blank=True,
+    )
+    external_id = models.CharField(max_length=64, blank=True)
+    full_name = models.CharField(max_length=255)
+    clone_url = models.URLField(blank=True)
+    default_branch = models.CharField(max_length=255, default="main")
+    enabled = models.BooleanField(default=False)
+    # Sync new commits on the branch without pressing Resync.
+    auto_sync = models.BooleanField(default=True)
+    # Tasks may push a branch and open a draft pull request. Owners decide.
+    allow_pull_requests = models.BooleanField(default=False)
+    # Run CI on that branch at once. Off by default, because the patch is
+    # untrusted until a person reviews it.
+    pull_request_ci = models.BooleanField(default=False)
+    # Jobs may open the issues their agent proposes. Owners decide.
+    allow_issues = models.BooleanField(default=False)
+    config_path = models.CharField(max_length=255, default=".secondshift.yaml")
+    # The commit runs use. It moves only when a sync succeeds.
+    config_sha = models.CharField(max_length=64, blank=True)
+    # The newest commit a sync read, even if its configuration was invalid.
+    checked_sha = models.CharField(max_length=64, blank=True)
+    last_synced_at = models.DateTimeField(null=True, blank=True)
+    last_sync_error = models.CharField(max_length=SYNC_ERROR_MAX_CHARS, blank=True)
+    removed_at = models.DateTimeField(null=True, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["workspace", "full_name"],
+                name="unique_workspace_repository",
+            ),
+            models.UniqueConstraint(
+                fields=["workspace", "local_path"],
+                condition=~models.Q(local_path=""),
+                name="unique_workspace_local_repository",
+            ),
+        ]
+        ordering = ["full_name"]
+        verbose_name_plural = "repositories"
+
+    def __str__(self) -> str:
+        return self.display_name
+
+    @property
+    def accepts_runs(self) -> bool:
+        return self.enabled and self.removed_at is None
+
+    @property
+    def receives_pull_requests(self) -> bool:
+        return self.source_type == self.SourceType.GITHUB and self.allow_pull_requests
+
+    @property
+    def receives_issues(self) -> bool:
+        return self.source_type == self.SourceType.GITHUB and self.allow_issues
+
+    @property
+    def display_name(self) -> str:
+        if self.source_type == self.SourceType.LOCAL and self.local_path:
+            return Path(self.local_path).name
+        return self.full_name
+
+    def clean(self) -> None:
+        if self.source_type == self.SourceType.LOCAL and not self.local_path:
+            raise ValidationError(
+                {"local_path": "Local repositories require a filesystem path."}
+            )
+        if self.source_type != self.SourceType.LOCAL and self.local_path:
+            raise ValidationError(
+                {"local_path": "Only local repositories may have a local path."}
+            )

secondshift/repositories/pull_requests.py

@@ -0,0 +1,199 @@
+"""Publish a run's changes to GitHub as a draft pull request.
+
+This is the only code that asks GitHub for write access. Each token is
+limited to one repository and to contents and pull requests, and is used for
+one push and one request. It never reaches the sandbox or the model.
+
+The commit is built in the control plane's mirror from the pinned commit and
+the run's stored patch, so publishing again never needs the agent or its
+sandbox. The author is the account that pushes: the App's bot, or the person
+who owns the token. With the same author, message and date, a retry makes the
+same commit and pushes the same branch again instead of making a new one.
+"""
+
+import os
+import subprocess
+import tempfile
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+from secondshift.connections.models import Connection
+from secondshift.github import credentials, pulls
+from secondshift.github.authors import GitAuthor, app_author, token_author
+from secondshift.github.errors import GitHubError
+from secondshift.github.git import authenticated_git
+from secondshift.repositories.github_access import github_access
+from secondshift.repositories.github_sync import fetch_github_commit
+from secondshift.repositories.mirrors import has_commit, locked_mirror
+from secondshift.repositories.models import Repository
+
+APP_PUSH_HINT = (
+    "Changes to workflow files are refused, and so is a branch someone else changed."
+)
+TOKEN_PUSH_HINT = (
+    "Give the token Contents and Pull requests write access. A branch someone "
+    "else changed is refused too, and so are workflow files unless the token "
+    "has Workflows write access."
+)
+
+
+class PublishError(Exception):
+    """A publishing step failed. The message is safe to show on the run."""
+
+
+@dataclass(frozen=True)
+class DraftPullRequest:
+    branch: str
+    base: str
+    title: str
+    body: str
+    message: str
+    committed_at: datetime
+
+
+@dataclass(frozen=True)
+class PublishedPullRequest:
+    number: int
+    url: str
+
+
+def publish_pull_request(
+    repository: Repository, commit_sha: str, patch: Path, draft: DraftPullRequest
+) -> PublishedPullRequest:
+    if repository.source_type != Repository.SourceType.GITHUB:
+        raise PublishError("Only GitHub repositories can receive pull requests.")
+    with_token = (
+        repository.github_connection is not None
+        and repository.github_connection.auth_strategy
+        == Connection.AuthStrategy.API_KEY
+    )
+    try:
+        # One token for the fetch, the push and the pull request.
+        access = github_access(repository, credentials.WRITE)
+    except GitHubError as error:
+        if with_token:
+            # Token errors already say what to fix, such as an expired token.
+            raise PublishError(str(error)) from error
+        raise PublishError(
+            "GitHub refused write access. Give the GitHub App Contents and Pull "
+            "requests write permission, then retry."
+        ) from error
+    try:
+        author = (token_author if with_token else app_author)(access.token)
+    except GitHubError as error:
+        raise PublishError(
+            "GitHub did not say which account publishes the changes."
+        ) from error
+    with locked_mirror(repository.pk) as mirror:
+        if not has_commit(mirror, commit_sha):
+            try:
+                fetch_github_commit(access, commit_sha, mirror)
+            except GitHubError as error:
+                # These messages never contain the token.
+                raise PublishError(str(error)) from error
+        commit = commit_patch(mirror, commit_sha, patch, draft, author)
+        try:
+            authenticated_git(
+                mirror,
+                access.token,
+                "push",
+                "--quiet",
+                "--",
+                access.clone_url,
+                f"{commit}:refs/heads/{draft.branch}",
+            )
+        except GitHubError as error:
+            raise PublishError(
+                f"GitHub refused the push to {draft.branch}. "
+                + (TOKEN_PUSH_HINT if with_token else APP_PUSH_HINT)
+            ) from error
+    return open_draft(access.full_name, access.token, draft)
+
+
+def commit_patch(
+    mirror: Path,
+    commit_sha: str,
+    patch: Path,
+    draft: DraftPullRequest,
+    author: GitAuthor,
+) -> str:
+    """Apply the patch to the pinned commit in a scratch index, and commit it."""
+    with tempfile.TemporaryDirectory(prefix="secondshift-publish-") as folder:
+        index = str(Path(folder) / "index")
+        git(mirror, index, draft, author, "read-tree", commit_sha)
+        try:
+            git(
+                mirror,
+                index,
+                draft,
+                author,
+                "apply",
+                "--cached",
+                "--binary",
+                str(patch),
+            )
+        except PublishError as error:
+            raise PublishError(
+                "The changes no longer apply to the run's commit."
+            ) from error
+        tree = git(mirror, index, draft, author, "write-tree")
+        return git(
+            mirror,
+            index,
+            draft,
+            author,
+            "commit-tree",
+            tree,
+            "-p",
+            commit_sha,
+            "-m",
+            draft.message,
+        )
+
+
+def git(
+    mirror: Path, index: str, draft: DraftPullRequest, author: GitAuthor, *args: str
+) -> str:
+    when = draft.committed_at.isoformat()
+    environment = {
+        "PATH": os.environ.get("PATH", "/usr/bin:/bin"),
+        "HOME": str(mirror),
+        "LC_ALL": "C",
+        "GIT_CONFIG_NOSYSTEM": "1",
+        "GIT_CONFIG_GLOBAL": os.devnull,
+        "GIT_INDEX_FILE": index,
+        "GIT_AUTHOR_NAME": author.name,
+        "GIT_AUTHOR_EMAIL": author.email,
+        "GIT_AUTHOR_DATE": when,
+        "GIT_COMMITTER_NAME": author.name,
+        "GIT_COMMITTER_EMAIL": author.email,
+        "GIT_COMMITTER_DATE": when,
+    }
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(mirror), "-c", "core.hooksPath=/dev/null", *args],
+            env=environment,
+            capture_output=True,
+            text=True,
+            timeout=120,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired) as error:
+        raise PublishError("The commit could not be built.") from error
+    if result.returncode:
+        raise PublishError("The commit could not be built.")
+    return result.stdout.strip()
+
+
+def open_draft(
+    full_name: str, token: str, draft: DraftPullRequest
+) -> PublishedPullRequest:
+    try:
+        opened = pulls.open_draft(
+            token, full_name, draft.title, draft.branch, draft.base, draft.body
+        )
+    except GitHubError as error:
+        # These messages never contain the token.
+        raise PublishError(str(error)) from error
+    return PublishedPullRequest(number=opened["number"], url=opened["html_url"])

secondshift/repositories/schemas.py

@@ -0,0 +1,102 @@
+from datetime import datetime
+
+from ninja import Schema
+from pydantic import Field
+
+from secondshift.repositories.status import ValidationStatus
+
+
+class RepositorySchema(Schema):
+    id: int
+    full_name: str
+    display_name: str
+    default_branch: str
+    enabled: bool
+    job_count: int
+    last_synced_at: datetime | None
+    source_type: str
+    local_path: str | None
+    revision: str
+    # The GitHub source connection it syncs through. None for local folders.
+    github_connection_id: int | None
+
+
+class LocalRepositoryCreateSchema(Schema):
+    path: str = Field(min_length=1, max_length=1000)
+
+
+class LocalRepositoryDirectorySchema(Schema):
+    name: str
+    path: str
+
+
+class LocalRepositoryDirectoryListSchema(Schema):
+    path: str
+    parent: str | None
+    directories: list[LocalRepositoryDirectorySchema]
+
+
+class RepositoryEnabledSchema(Schema):
+    enabled: bool
+
+
+class RepositoryPullRequestsSchema(Schema):
+    allow: bool
+    run_ci: bool = False
+
+
+class RepositoryIssuesSchema(Schema):
+    allow: bool
+
+
+class RepositoryValidationSchema(Schema):
+    status: ValidationStatus
+    message: str
+    current_revision: str | None
+    synced_revision: str
+    needs_sync: bool
+
+
+class RepositoryJobSchema(Schema):
+    id: int
+    name: str
+    skill_path: str
+    cron: str
+    timezone: str
+    enabled: bool
+    grant_status: str
+    removed_at: datetime | None = None
+
+
+class RepositoryDetailSchema(RepositorySchema):
+    config_path: str
+    # The repository's choice, and whether this server syncs in the background.
+    auto_sync: bool
+    server_syncs: bool
+    # GitHub repositories only. See RepositoryPullRequestsSchema.
+    can_open_pull_requests: bool
+    allow_pull_requests: bool
+    pull_request_ci: bool
+    # GitHub repositories only, like pull requests.
+    allow_issues: bool
+    validation: RepositoryValidationSchema
+    jobs: list[RepositoryJobSchema]
+
+
+class GitHubRepositoryCreateSchema(Schema):
+    connection_id: int
+    repository_id: int = Field(gt=0)
+    branch: str = Field(default="", max_length=255)
+
+
+class GitHubAvailableRepositorySchema(Schema):
+    connection_id: int
+    repository_id: int
+    full_name: str
+    default_branch: str
+
+
+class GitHubAppStatusSchema(Schema):
+    configured: bool
+    error: str = ""
+    repositories: list[GitHubAvailableRepositorySchema]

secondshift/repositories/serialization.py

@@ -0,0 +1,133 @@
+from django.http import HttpRequest
+
+from secondshift.github import repos
+from secondshift.github.errors import GitHubError
+from secondshift.jobs.autosync import server_syncs
+from secondshift.jobs.dao import repository_jobs_with_grants
+from secondshift.jobs.repository_sync import (
+    RepositoryConfigError,
+    inspect_local_repository,
+)
+from secondshift.repositories.github_access import github_access
+from secondshift.repositories.local import (
+    LocalRepositoryError,
+)
+from secondshift.repositories.models import Repository
+from secondshift.repositories.schemas import (
+    RepositoryDetailSchema,
+    RepositoryJobSchema,
+    RepositorySchema,
+    RepositoryValidationSchema,
+)
+from secondshift.repositories.status import ValidationStatus, source_status
+from secondshift.workspaces.access import (
+    Action,
+    allows,
+)
+
+
+def visible_local_path(request: HttpRequest, repository: Repository) -> str | None:
+    if repository.source_type == Repository.SourceType.LOCAL and allows(
+        request, Action.VIEW_LOCAL_PATHS
+    ):
+        return repository.local_path
+    return None
+
+
+def serialize_repository(
+    request: HttpRequest,
+    repository: Repository,
+) -> RepositorySchema:
+    return RepositorySchema(
+        id=repository.pk,
+        full_name=repository.full_name,
+        display_name=repository.display_name,
+        default_branch=repository.default_branch,
+        enabled=repository.enabled,
+        # The list endpoint prefetches jobs, so this reads them from memory.
+        job_count=len(repository.jobs.all()),
+        last_synced_at=repository.last_synced_at,
+        source_type=repository.source_type,
+        local_path=visible_local_path(request, repository),
+        revision=repository.config_sha,
+        github_connection_id=repository.github_connection_id,
+    )
+
+
+def validation_schema(
+    repository: Repository, head: str | None, problem: str
+) -> RepositoryValidationSchema:
+    """`head` is None when the branch could not be read."""
+    status = source_status(repository, head, problem)
+    return RepositoryValidationSchema(
+        status=status.status,
+        message=status.message,
+        current_revision=head,
+        synced_revision=repository.config_sha,
+        needs_sync=status.needs_sync,
+    )
+
+
+def unreachable(repository: Repository, error: Exception) -> RepositoryValidationSchema:
+    return RepositoryValidationSchema(
+        status=ValidationStatus.ERROR,
+        message=str(error),
+        current_revision=None,
+        synced_revision=repository.config_sha,
+        needs_sync=False,
+    )
+
+
+def validate_repository(repository: Repository) -> RepositoryValidationSchema:
+    """Read the branch head, then report the state the last syncs saved."""
+    if repository.source_type != Repository.SourceType.LOCAL:
+        try:
+            access = github_access(repository)
+            head = repos.branch_head(
+                access.token, access.repository_id, repository.default_branch
+            )
+        except GitHubError as error:
+            return unreachable(repository, error)
+        return validation_schema(repository, head, repository.last_sync_error)
+    try:
+        metadata = inspect_local_repository(
+            repository.local_path, repository.default_branch
+        )
+    except RepositoryConfigError as error:
+        # The branch is readable, but its configuration is broken right now.
+        return validation_schema(repository, None, str(error))
+    except LocalRepositoryError as error:
+        return unreachable(repository, error)
+    return validation_schema(repository, metadata.revision, repository.last_sync_error)
+
+
+def serialize_repository_detail(
+    request: HttpRequest,
+    repository: Repository,
+) -> RepositoryDetailSchema:
+    summary = serialize_repository(request, repository)
+    jobs = repository_jobs_with_grants(repository)
+    return RepositoryDetailSchema(
+        **summary.model_dump(),
+        config_path=repository.config_path,
+        auto_sync=repository.auto_sync,
+        server_syncs=server_syncs(),
+        can_open_pull_requests=repository.source_type == Repository.SourceType.GITHUB,
+        allow_pull_requests=repository.allow_pull_requests,
+        pull_request_ci=repository.pull_request_ci,
+        allow_issues=repository.allow_issues,
+        validation=validate_repository(repository),
+        jobs=[
+            RepositoryJobSchema(
+                id=job.pk,
+                name=job.name,
+                skill_path=job.skill_path,
+                cron=job.cron,
+                timezone=job.timezone,
+                enabled=job.enabled,
+                grant_status=job.capability_grant_status(),
+                removed_at=job.removed_at,
+            )
+            for job in jobs
+        ],
+    )

secondshift/repositories/skill_source.py

@@ -0,0 +1,64 @@
+"""Resolve a submitted skill at a committed revision without syncing jobs."""
+
+import re
+from pathlib import Path
+
+from django.conf import settings
+
+from secondshift.config.constants import COMMIT_SHA_PATTERN
+from secondshift.github.git import authenticated_git
+from secondshift.repositories.github_access import github_access
+from secondshift.repositories.local import committed_file, git
+from secondshift.repositories.mirrors import locked_mirror
+from secondshift.repositories.models import Repository
+
+
+class SkillSourceError(ValueError):
+    pass
+
+
+def skill_commit(
+    repository: Repository, skill_path: str | None, revision: str | None
+) -> str:
+    """Use the followed branch by default; explicit branches and full SHAs work."""
+    if (
+        repository.source_type == Repository.SourceType.LOCAL
+        and not settings.LOCAL_REPOSITORIES_ENABLED
+    ):
+        raise SkillSourceError("Local repositories are not enabled on this server.")
+    requested = revision or repository.default_branch
+    is_commit = bool(re.fullmatch(COMMIT_SHA_PATTERN, requested))
+    if not is_commit:
+        # Only branch names are accepted, never arbitrary Git revision expressions.
+        git(
+            Path(repository.local_path) if repository.local_path else Path.cwd(),
+            "check-ref-format",
+            f"refs/heads/{requested}",
+        )
+    if repository.source_type == Repository.SourceType.LOCAL:
+        root = Path(repository.local_path)
+        ref = requested if is_commit else f"refs/heads/{requested}"
+        commit = git(root, "rev-parse", "--verify", f"{ref}^{{commit}}")
+        if skill_path is not None:
+            committed_file(root, commit, skill_path)
+        return commit
+    access = github_access(repository)
+    token = access.token
+    with locked_mirror(repository.pk) as mirror:
+        authenticated_git(
+            mirror,
+            token,
+            "fetch",
+            "--no-tags",
+            "--",
+            access.clone_url,
+            requested if is_commit else f"refs/heads/{requested}",
+        )
+        commit = authenticated_git(
+            mirror, token, "rev-parse", "--verify", "FETCH_HEAD^{commit}"
+        )
+        if not re.fullmatch(COMMIT_SHA_PATTERN, commit):
+            raise SkillSourceError("The repository did not resolve to a full commit.")
+        if skill_path is not None:
+            committed_file(mirror, commit, skill_path)
+        return commit