secondshift 0.1.0b12__py3-none-any.whl

secondshift/repositories/external_skills.py

@@ -0,0 +1,224 @@
+"""Skills from another public GitHub repository, pinned to one commit.
+
+A skill is a SKILL.md and the files in its folder. SecondShift fetches them
+without credentials, so only public repositories work. A run gets the folder
+at the pinned commit, mounted read-only at /skill. Nothing in the skill's
+repository can change how the sandbox runs or what it may reach.
+"""
+
+import fcntl
+import hashlib
+import os
+import posixpath
+import re
+import subprocess
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+from django.conf import settings
+
+from secondshift.config.constants import COMMIT_SHA_PATTERN, GITHUB_REPOSITORY_PATTERN
+from secondshift.github.errors import GitHubError
+from secondshift.github.git import public_git
+from secondshift.repositories.local import LocalRepositoryError, committed_file
+from secondshift.repositories.mirrors import has_commit
+from secondshift.repositories.snapshots import (
+    SKILL_LIMITS,
+    SnapshotError,
+    tree_entries,
+)
+from secondshift.repositories.sources import (
+    SourceSnapshot,
+    file_sha256,
+    skill_mirrors_root,
+    skill_snapshots_root,
+    write_atomically,
+)
+
+SKILL_FILE = "SKILL.md"
+# Branch and tag names a person may give. Git checks them again.
+REF_NAME = re.compile(r"[A-Za-z0-9._][A-Za-z0-9._/-]{0,199}")
+
+
+class ExternalSkillError(ValueError):
+    """Worded for the person who named the skill."""
+
+
+@dataclass(frozen=True)
+class ExternalSkill:
+    repository: str
+    path: str
+    commit_sha: str
+    text: str
+    file_count: int
+    size_bytes: int
+
+
+def check_reference(repository: str, path: str) -> None:
+    if not re.fullmatch(GITHUB_REPOSITORY_PATTERN, repository) or any(
+        part in {".", ".."} for part in repository.split("/")
+    ):
+        raise ExternalSkillError("Name the repository as owner/name.")
+    pure = PurePosixPath(path)
+    if (
+        pure.is_absolute()
+        or ".." in pure.parts
+        or "\0" in path
+        or posixpath.normpath(path) != path
+    ):
+        raise ExternalSkillError("The skill path must be a path inside its repository.")
+    if pure.name != SKILL_FILE:
+        raise ExternalSkillError(f"The skill path must point to a {SKILL_FILE} file.")
+
+
+def check_ref(ref: str) -> None:
+    if re.fullmatch(COMMIT_SHA_PATTERN, ref):
+        return
+    if (
+        not REF_NAME.fullmatch(ref)
+        or ".." in ref
+        or "//" in ref
+        or ref.endswith(("/", ".lock", "."))
+        or "@{" in ref
+    ):
+        raise ExternalSkillError("Give a branch, a tag or a full commit SHA.")
+    completed = subprocess.run(
+        ["git", "check-ref-format", f"refs/heads/{ref}"],
+        capture_output=True,
+        check=False,
+        timeout=10,
+    )
+    if completed.returncode:
+        raise ExternalSkillError("Give a branch, a tag or a full commit SHA.")
+
+
+def skill_folder(path: str) -> str:
+    """The folder whose files a run gets. Empty for a SKILL.md at the root."""
+    return posixpath.dirname(path)
+
+
+def cache_key(repository: str) -> str:
+    # Repository names are case-insensitive on GitHub.
+    return hashlib.sha256(repository.lower().encode()).hexdigest()[:32]
+
+
+@contextmanager
+def locked_skill_mirror(repository: str) -> Iterator[Path]:
+    """A bare mirror of the skill's repository, held under a lock."""
+    key = cache_key(repository)
+    path = skill_mirrors_root() / f"{key}.git"
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    locks = settings.SOURCE_CACHE_ROOT / "locks"
+    locks.mkdir(parents=True, exist_ok=True, mode=0o700)
+    with (locks / f"skill-{key}.lock").open("a+b") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        if not path.exists():
+            subprocess.run(
+                ["git", "init", "--quiet", "--bare", str(path)],
+                check=True,
+                capture_output=True,
+                timeout=30,
+            )
+        # Mark it used, so cache cleanup keeps it.
+        os.utime(path)
+        yield path
+
+
+def clone_url(repository: str) -> str:
+    return f"{settings.EXTERNAL_SKILLS_BASE_URL}/{repository}.git"
+
+
+def fetch(mirror: Path, repository: str, ref: str) -> str:
+    """Fetch one branch, tag or commit and return its full commit SHA."""
+    try:
+        public_git(
+            mirror, "fetch", "--no-tags", "--depth=1", "--", clone_url(repository), ref
+        )
+        commit = public_git(mirror, "rev-parse", "--verify", "FETCH_HEAD^{commit}")
+    except GitHubError as error:
+        raise ExternalSkillError(
+            f"Could not fetch {ref} from {repository}. Only public GitHub "
+            "repositories work, and the branch, tag or commit must exist."
+        ) from error
+    if not re.fullmatch(COMMIT_SHA_PATTERN, commit):
+        raise ExternalSkillError(f"{repository} did not resolve to a full commit.")
+    # Keep the commit's objects in the mirror until cache cleanup.
+    public_git(mirror, "update-ref", f"refs/secondshift/skills/{commit}", commit)
+    return commit
+
+
+def mirrored_commit(mirror: Path, repository: str, ref: str) -> str:
+    if re.fullmatch(COMMIT_SHA_PATTERN, ref) and has_commit(mirror, ref):
+        return ref
+    commit = fetch(mirror, repository, ref)
+    if re.fullmatch(COMMIT_SHA_PATTERN, ref) and commit != ref:
+        raise ExternalSkillError(f"{repository} did not return commit {ref[:12]}.")
+    return commit
+
+
+def require_enabled() -> None:
+    if not settings.EXTERNAL_SKILLS_ENABLED:
+        raise ExternalSkillError(
+            "This server does not run skills from other repositories."
+        )
+
+
+def resolve_external_skill(
+    repository: str, path: str, ref: str | None = None
+) -> ExternalSkill:
+    """Fetch the skill at a branch, tag or commit, and pin it to a commit.
+
+    Without a ref, the repository's default branch is used.
+    """
+    require_enabled()
+    check_reference(repository, path)
+    requested = ref or "HEAD"
+    if ref is not None:
+        check_ref(ref)
+    with locked_skill_mirror(repository) as mirror:
+        commit = mirrored_commit(mirror, repository, requested)
+        try:
+            text = committed_file(mirror, commit, path)
+        except LocalRepositoryError as error:
+            raise ExternalSkillError(
+                f"{path} is not a file in {repository} at {commit[:12]}."
+            ) from error
+        try:
+            entries = tree_entries(mirror, commit, skill_folder(path))
+        except SnapshotError as error:
+            raise ExternalSkillError(str(error)) from error
+    return ExternalSkill(
+        repository=repository,
+        path=path,
+        commit_sha=commit,
+        text=text,
+        file_count=len(entries),
+        size_bytes=sum(entry.size for entry in entries),
+    )
+
+
+def skill_snapshot(repository: str, path: str, commit_sha: str) -> SourceSnapshot:
+    """The skill folder at a commit, as a snapshot a run can unpack.
+
+    Built and cached if needed. Raises SnapshotError or ExternalSkillError.
+    """
+    require_enabled()
+    check_reference(repository, path)
+    if not re.fullmatch(COMMIT_SHA_PATTERN, commit_sha):
+        raise SnapshotError("A full commit SHA is required.")
+    name = hashlib.sha256(f"{repository.lower()}\0{path}".encode()).hexdigest()[:32]
+    target = skill_snapshots_root() / f"{commit_sha}-{name}.tar.gz"
+    if target.exists():
+        target.touch()
+    else:
+        target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        with locked_skill_mirror(repository) as mirror:
+            mirrored_commit(mirror, repository, commit_sha)
+            write_atomically(
+                mirror, commit_sha, target, skill_folder(path), SKILL_LIMITS
+            )
+    return SourceSnapshot(
+        path=target, sha256=file_sha256(target), size_bytes=target.stat().st_size
+    )

secondshift/repositories/github.py

@@ -0,0 +1,27 @@
+"""Resolve GitHub scope from server-managed repository identity."""
+
+from secondshift.connections.repository_ref import RepositoryRef
+from secondshift.github.identity import github_remote_name, valid_name
+from secondshift.repositories.models import Repository
+
+
+def github_repository_scope(repository: Repository) -> str | None:
+    if repository.source_type == Repository.SourceType.LOCAL:
+        return github_remote_name(repository.clone_url)
+    name = repository.full_name
+    return name if valid_name(name) else None
+
+
+def repository_ref(repository: Repository) -> RepositoryRef:
+    """What external tools may know about the repository a job works on."""
+    from_github = repository.source_type == Repository.SourceType.GITHUB
+    return RepositoryRef(
+        workspace_id=repository.workspace_id,
+        github_name=github_repository_scope(repository),
+        github_id=(
+            int(repository.external_id)
+            if from_github and repository.external_id
+            else None
+        ),
+        source=repository.github_connection if from_github else None,
+    )

secondshift/repositories/github_access.py

@@ -0,0 +1,57 @@
+"""Reach a repository on GitHub through the connection it was added with.
+
+Each operation gets one token and checks the repository's identity once, then
+uses that token for every call and git command it makes.
+"""
+
+from dataclasses import dataclass
+
+from secondshift.connections.github import connection_source
+from secondshift.connections.models import Connection
+from secondshift.github import credentials, repos
+from secondshift.github.errors import GitHubError
+from secondshift.github.identity import clone_url
+from secondshift.repositories.models import Repository
+
+
+@dataclass(frozen=True)
+class GitHubAccess:
+    """A checked repository and a token for it. Never stored or logged."""
+
+    repository_id: int
+    full_name: str
+    token: str
+    metadata: dict
+
+    @property
+    def clone_url(self) -> str:
+        return clone_url(self.full_name)
+
+
+def connection_access(
+    connection: Connection | None,
+    repository_id: int,
+    permissions: dict[str, str] = credentials.READ,
+) -> GitHubAccess:
+    """Access to a repository the connection reaches, before it has a row."""
+    token = credentials.token_for(
+        connection_source(connection), repository_id, permissions
+    )
+    metadata = repos.repository(token, repository_id)
+    return GitHubAccess(repository_id, metadata["full_name"], token, metadata)
+
+
+def github_access(
+    repository: Repository, permissions: dict[str, str] = credentials.READ
+) -> GitHubAccess:
+    """Access to an added repository. Fails if it was removed, moved or renamed."""
+    if repository.removed_at is not None:
+        raise GitHubError("This repository has been removed.")
+    connection = repository.github_connection
+    if connection is not None and connection.workspace_id != repository.workspace_id:
+        raise GitHubError("The GitHub connection is disconnected.")
+    access = connection_access(connection, int(repository.external_id), permissions)
+    # Do not silently move a repository's identity or its existing approvals.
+    if repository.full_name != access.full_name:
+        raise GitHubError("Repository was renamed. Remove and import it again.")
+    return access

secondshift/repositories/github_check.py

@@ -0,0 +1,279 @@
+"""Check what a GitHub source connection can still do, feature by feature.
+
+The result is kept as the connection's last check, so the dashboard shows
+each gap with a link to fix it on GitHub. Tokens are never part of it.
+"""
+
+from dataclasses import asdict, dataclass
+from datetime import datetime, timedelta
+
+from django.utils import timezone
+
+from secondshift.config.constants import GITHUB_BASE_URL
+from secondshift.connections.checks import record_check
+from secondshift.connections.dao import connected_github_sources
+from secondshift.connections.github import can_check, connection_source
+from secondshift.connections.models import Connection
+from secondshift.github import app, credentials, permissions
+from secondshift.github.errors import GitHubError
+from secondshift.github.tokens import inspect_token
+from secondshift.repositories.dao import connection_repositories
+from secondshift.repositories.github_access import (
+    GitHubAccess,
+    github_access,
+)
+from secondshift.repositories.models import Repository
+
+# A grant can be withdrawn on GitHub at any time, so each source is checked daily.
+CHECK_INTERVAL = timedelta(days=1)
+
+GITHUB_TOKENS_URL = f"{GITHUB_BASE_URL}/settings/personal-access-tokens"
+
+PERMISSION_LABELS = {
+    "contents": "Contents",
+    "pull_requests": "Pull requests",
+    "actions": "Actions",
+    "statuses": "Commit statuses",
+    "issues": "Issues",
+}
+
+
+@dataclass(frozen=True)
+class FeatureCheck:
+    feature: str
+    label: str
+    # None when it cannot be checked without using it.
+    ok: bool | None
+    message: str
+
+
+@dataclass(frozen=True)
+class SourceCheck:
+    ok: bool
+    message: str
+    features: list[FeatureCheck]
+    fix_url: str = ""
+
+
+def names(permission_names: list[str]) -> str:
+    return ", ".join(PERMISSION_LABELS[name] for name in permission_names)
+
+
+def features(
+    reads: dict[str, bool] | None, writes: dict[str, bool] | None
+) -> list[FeatureCheck]:
+    """One line per feature. reads is None when no repository could be probed."""
+    if reads is None:
+        unknown = "Add a repository to check this."
+        return [
+            FeatureCheck("sync", "Sync", None, unknown),
+            FeatureCheck("tools", "GitHub tools", None, unknown),
+            FeatureCheck("draft_pull_requests", "Draft pull requests", None, unknown),
+            FeatureCheck("issues", "Issues", None, unknown),
+        ]
+    missing_tools = [name for name in permissions.TOOLS if not reads[name]]
+    sync = (
+        FeatureCheck("sync", "Sync", True, "Can read code.")
+        if reads["contents"]
+        else FeatureCheck(
+            "sync", "Sync", False, "Needs read access to Contents. Sync fails."
+        )
+    )
+    tools = (
+        FeatureCheck(
+            "tools",
+            "GitHub tools",
+            True,
+            "Can read pull requests, Actions runs and commit statuses.",
+        )
+        if not missing_tools
+        else FeatureCheck(
+            "tools",
+            "GitHub tools",
+            False,
+            f"Needs read access to {names(missing_tools)}.",
+        )
+    )
+    drafts = write_feature(
+        writes,
+        "draft_pull_requests",
+        "Draft pull requests",
+        permissions.DRAFT_PULL_REQUESTS,
+        "Can push a branch and open a draft pull request.",
+        "a draft pull request opens",
+    )
+    issues = write_feature(
+        writes,
+        "issues",
+        "Issues",
+        permissions.ISSUES,
+        "Can open issues.",
+        "an issue opens",
+    )
+    return [sync, tools, drafts, issues]
+
+
+def write_feature(
+    writes: dict[str, bool] | None,
+    feature: str,
+    label: str,
+    needed: tuple[str, ...],
+    ready: str,
+    first_use: str,
+) -> FeatureCheck:
+    """A feature that writes. A token's write access is only known on use."""
+    if writes is None:
+        return FeatureCheck(
+            feature,
+            label,
+            None,
+            f"A token's write access is checked when {first_use}.",
+        )
+    missing = [name for name in needed if not writes[name]]
+    if missing:
+        return FeatureCheck(
+            feature, label, False, f"Needs write access to {names(missing)}."
+        )
+    return FeatureCheck(feature, label, True, ready)
+
+
+def reach(repositories: list[Repository]) -> tuple[GitHubAccess | None, list[str]]:
+    """The first repository reached, and the names of those that were not."""
+    first: GitHubAccess | None = None
+    unreachable: list[str] = []
+    for repository in repositories:
+        try:
+            access = github_access(repository)
+        except GitHubError:
+            unreachable.append(repository.full_name)
+            continue
+        first = first or access
+    return first, unreachable
+
+
+def summary(
+    repositories: list[Repository], unreachable: list[str], checked: list[FeatureCheck]
+) -> tuple[bool, str]:
+    if unreachable:
+        return False, (
+            f"Cannot reach {', '.join(unreachable)}. "
+            "Give this connection access to them on GitHub."
+        )
+    if checked[0].ok is False:
+        return False, "Cannot read code, so sync fails."
+    if not repositories:
+        return True, "Connected. No repositories added yet."
+    count = len(repositories)
+    return (
+        True,
+        f"Reaches {'its repository' if count == 1 else f'all {count} repositories'}.",
+    )
+
+
+def check_token(
+    connection: Connection,
+    source: credentials.PersonalToken,
+    repositories: list[Repository],
+) -> SourceCheck:
+    owner = inspect_token(source.token)
+    if connection.expires_at != owner.expires_at:
+        connection.expires_at = owner.expires_at
+        connection.save(update_fields=["expires_at", "updated_at"])
+    first, unreachable = reach(repositories)
+    reads = (
+        permissions.token_reads(
+            source.token,
+            first.full_name,
+            str(first.metadata.get("default_branch") or "HEAD"),
+            permissions.SYNC + permissions.TOOLS,
+        )
+        if first
+        else None
+    )
+    checked = features(reads, None)
+    ok, message = summary(repositories, unreachable, checked)
+    return SourceCheck(ok, message, checked, GITHUB_TOKENS_URL)
+
+
+def check_installation(
+    source: credentials.AppInstallation, repositories: list[Repository]
+) -> SourceCheck:
+    try:
+        details = app.installation(source.installation_id)
+    except GitHubError:
+        return SourceCheck(
+            False,
+            "GitHub could not find this App installation. It may have been removed.",
+            [],
+        )
+    granted = details.permissions
+    # An App's grants hold for every repository, so no repository is probed.
+    unreachable = reach(repositories)[1]
+    checked = features(
+        permissions.granted_reads(granted, permissions.SYNC + permissions.TOOLS),
+        permissions.granted_writes(
+            granted, permissions.DRAFT_PULL_REQUESTS + permissions.ISSUES
+        ),
+    )
+    ok, message = summary(repositories, unreachable, checked)
+    return SourceCheck(ok, message, checked, details.settings_url)
+
+
+def check_source(connection: Connection) -> SourceCheck:
+    """Check the credential, every added repository, and each feature's access."""
+    repositories = connection_repositories(connection)
+    try:
+        source = connection_source(connection)
+        if isinstance(source, credentials.AppInstallation) and not app.configured():
+            # Without the App's key, no installation token can be made.
+            raise GitHubError(
+                "This server has no GitHub App configured, so this connection "
+                "cannot sync. Ask the server's operator to configure the App, "
+                "or connect a personal access token instead."
+            )
+        if isinstance(source, credentials.PersonalToken):
+            result = check_token(connection, source, repositories)
+        else:
+            result = check_installation(source, repositories)
+    except GitHubError as error:
+        # These messages never contain a token.
+        token = connection.auth_strategy == Connection.AuthStrategy.API_KEY
+        result = SourceCheck(False, str(error), [], GITHUB_TOKENS_URL if token else "")
+    record_check(
+        connection,
+        result.ok,
+        result.message,
+        features=[asdict(feature) for feature in result.features],
+        fix_url=result.fix_url,
+    )
+    return result
+
+
+def last_checked(connection: Connection) -> datetime | None:
+    check = connection.last_check or {}
+    try:
+        return datetime.fromisoformat(check["checked_at"])
+    except (KeyError, TypeError, ValueError):
+        return None
+
+
+def check_due_sources() -> int:
+    """Check each GitHub source not checked within a day. Returns how many."""
+    due = timezone.now() - CHECK_INTERVAL
+    checked = 0
+    for connection in connected_github_sources():
+        when = last_checked(connection)
+        if not can_check(connection) or (when is not None and when > due):
+            continue
+        check_source(connection)
+        checked += 1
+    return checked
+
+
+def check_after_failure(connection: Connection | None) -> None:
+    """A sync failed, so show why on the connection, unless it already shows a failure."""
+    if not can_check(connection):
+        return
+    if connection.failed_last_check:
+        return
+    check_source(connection)