secondshift 0.1.0b12__py3-none-any.whl

secondshift/repositories/github_sync.py

@@ -0,0 +1,181 @@
+"""Fetch GitHub source into the control plane's mirror. Tokens never leave it."""
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+
+from secondshift.connections.dao import connected_github_sources
+from secondshift.connections.github import connection_source, usable_source
+from secondshift.connections.models import Connection
+from secondshift.github import credentials, repos
+from secondshift.github.app import configured, installation_repositories
+from secondshift.github.errors import GitHubError
+from secondshift.github.git import authenticated_git
+from secondshift.github.identity import clone_url
+from secondshift.repositories.dao import (
+    lock_or_create_github_repository,
+    lock_repository,
+)
+from secondshift.repositories.github_access import (
+    GitHubAccess,
+    connection_access,
+    github_access,
+)
+from secondshift.repositories.mirrors import SYNCED_REF, locked_mirror
+from secondshift.repositories.models import Repository
+from secondshift.workspaces.models import Workspace
+
+
+@dataclass(frozen=True)
+class FetchedBranch:
+    """The branch's newest commit, fetched into the mirror."""
+
+    mirror: Path
+    revision: str
+    token: str
+
+    def mark_synced(self) -> None:
+        """Keep this commit's objects in the mirror."""
+        authenticated_git(
+            self.mirror, self.token, "update-ref", SYNCED_REF, self.revision
+        )
+
+
+@contextmanager
+def fetched_branch(repository: Repository) -> Iterator[FetchedBranch]:
+    """Fetch the branch and hold the mirror's lock until the caller is done.
+
+    The fetch runs outside any database transaction.
+    """
+    access = github_access(repository)
+    with locked_mirror(repository.pk) as mirror:
+        authenticated_git(
+            mirror,
+            access.token,
+            "fetch",
+            "--no-tags",
+            "--",
+            access.clone_url,
+            f"refs/heads/{repository.default_branch}",
+        )
+        revision = authenticated_git(mirror, access.token, "rev-parse", "FETCH_HEAD")
+        yield FetchedBranch(mirror, revision, access.token)
+
+
+def record_synced_revision(repository_id: int, revision: str) -> Repository:
+    """Save a good sync of this commit. Call inside a transaction."""
+    locked = lock_repository(repository_id)
+    locked.config_sha = revision
+    locked.checked_sha = revision
+    locked.last_sync_error = ""
+    locked.save(
+        update_fields=["config_sha", "checked_sha", "last_sync_error", "updated_at"]
+    )
+    return locked
+
+
+def fetch_github_commit(access: GitHubAccess, commit_sha: str, mirror: Path) -> None:
+    """Fetch one commit by its SHA into the repository's mirror."""
+    try:
+        authenticated_git(
+            mirror,
+            access.token,
+            "fetch",
+            "--no-tags",
+            "--",
+            access.clone_url,
+            commit_sha,
+        )
+    except GitHubError as error:
+        raise GitHubError(
+            f"The commit {commit_sha[:12]} is no longer available from GitHub."
+        ) from error
+
+
+class GitHubRepositoryExistsError(GitHubError):
+    pass
+
+
+def source_repositories(connection: Connection) -> list[dict]:
+    """What a source connection can import, as GitHub lists it."""
+    source = connection_source(connection)
+    if isinstance(source, credentials.AppInstallation):
+        return installation_repositories(source.installation_id)
+    return repos.token_repositories(source.token)
+
+
+def available_github_sources(
+    workspace: Workspace,
+) -> tuple[list[dict[str, object]], list[str]]:
+    available = []
+    errors = []
+    for connection in connected_github_sources(workspace.pk):
+        if not usable_source(connection) or (
+            connection.auth_strategy == Connection.AuthStrategy.MANAGED
+            and not configured()
+        ):
+            continue
+        try:
+            repositories = source_repositories(connection)
+        except GitHubError as error:
+            errors.append(f"{connection.display_name}: {error}")
+            continue
+        for item in repositories:
+            available.append(
+                {
+                    "connection_id": connection.pk,
+                    "repository_id": item["id"],
+                    "full_name": item["full_name"],
+                    "default_branch": item["default_branch"],
+                }
+            )
+    return available, errors
+
+
+# A removed repository that is added again gets these back if its sync fails.
+RESTORED_FIELDS = (
+    "removed_at",
+    "github_connection",
+    "default_branch",
+)
+
+
+def save_attached_repository(
+    workspace: Workspace,
+    connection: Connection,
+    repository_id: int,
+    branch: str,
+    metadata: dict,
+) -> tuple[Repository, dict | None]:
+    """The new or restored row, and a restored row's earlier values."""
+    repository, created = lock_or_create_github_repository(
+        workspace,
+        metadata["full_name"],
+        defaults={
+            "source_type": Repository.SourceType.GITHUB,
+            "github_connection": connection,
+            "external_id": str(repository_id),
+            "clone_url": clone_url(metadata["full_name"]),
+            "default_branch": branch or metadata["default_branch"],
+        },
+    )
+    if created:
+        return repository, None
+    if (
+        repository.removed_at is None
+        or repository.source_type != Repository.SourceType.GITHUB
+        or repository.external_id != str(repository_id)
+    ):
+        raise GitHubRepositoryExistsError("This repository is already connected.")
+    previous = {field: getattr(repository, field) for field in RESTORED_FIELDS}
+    repository.removed_at = None
+    repository.github_connection = connection
+    repository.default_branch = branch or metadata["default_branch"]
+    repository.save()
+    return repository, previous
+
+
+def repository_branches(connection: Connection, repository_id: int) -> list[str]:
+    access = connection_access(connection, repository_id)
+    return repos.branches(access.token, access.full_name)

secondshift/repositories/github_views.py

@@ -0,0 +1,90 @@
+"""Browser authorization verifies installations with the authorizing user's token."""
+
+import secrets
+import time
+
+from django.conf import settings
+from django.contrib.auth.decorators import login_required
+from django.core.exceptions import PermissionDenied
+from django.http import HttpRequest, HttpResponse, HttpResponseBadRequest
+from django.shortcuts import redirect
+from django.urls import reverse
+from django.views.decorators.http import require_GET
+
+from secondshift.connections.github import link_installations
+from secondshift.github import installations
+from secondshift.github.app import configured
+from secondshift.github.errors import GitHubError
+from secondshift.workspaces.access import AccessDenied, Action, require
+
+STATE_KEY = "secondshift.github_installation"
+
+
+@login_required
+@require_GET
+def github_start(request: HttpRequest) -> HttpResponse:
+    try:
+        workspace = require(request, Action.ADD_GITHUB_REPOSITORIES)
+    except AccessDenied as error:
+        raise PermissionDenied(str(error)) from error
+    if not configured():
+        return redirect("/repositories/new?source=github")
+    state = secrets.token_urlsafe(32)
+    request.session[STATE_KEY] = {
+        "state": state,
+        "workspace": workspace.pk,
+        "time": time.time(),
+    }
+    if request.GET.get("install"):
+        return redirect(installations.installation_url(state))
+    return redirect(installations.authorization_url(state, callback_url()))
+
+
+@login_required
+@require_GET
+def github_callback(request: HttpRequest) -> HttpResponse:
+    try:
+        workspace = require(request, Action.ADD_GITHUB_REPOSITORIES)
+    except AccessDenied as error:
+        raise PermissionDenied(str(error)) from error
+    pending = request.session.get(STATE_KEY, {})
+    state = request.GET.get("state", "")
+    if (
+        not state
+        or not secrets.compare_digest(state, pending.get("state", ""))
+        or pending.get("workspace") != workspace.pk
+        or time.time() - pending.get("time", 0) > 600
+    ):
+        return HttpResponseBadRequest(
+            "GitHub authorization expired or does not match this workspace."
+        )
+    if request.GET.get("setup_action") == "request":
+        request.session.pop(STATE_KEY, None)
+        return HttpResponseBadRequest(
+            "Your organization must approve the GitHub App installation. Connect again after approval."
+        )
+    if request.GET.get("error"):
+        request.session.pop(STATE_KEY, None)
+        return HttpResponseBadRequest("GitHub authorization was cancelled.")
+    code = request.GET.get("code", "")
+    if not code:
+        # Installations that already exist still require user authorization.
+        return redirect(installations.authorization_url(state, callback_url()))
+    request.session.pop(STATE_KEY, None)
+    try:
+        token = installations.user_token(code, callback_url())
+        if not link_installations(workspace, installations.user_installations(token)):
+            state = secrets.token_urlsafe(32)
+            request.session[STATE_KEY] = {
+                "state": state,
+                "workspace": workspace.pk,
+                "time": time.time(),
+            }
+            return redirect(installations.installation_url(state))
+    except GitHubError as error:
+        return HttpResponseBadRequest(str(error))
+    return redirect("/repositories/new?source=github")
+
+
+def callback_url() -> str:
+    return settings.FRONTEND_URL.rstrip("/") + reverse("github-callback")

secondshift/repositories/local.py

@@ -0,0 +1,198 @@
+import hashlib
+import logging
+import posixpath
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from secondshift.config.constants import GITHUB_BASE_URL, MANIFEST_FILE
+from secondshift.repositories.dao import local_repository, save_local_repository
+from secondshift.repositories.github import github_remote_name
+from secondshift.repositories.models import SYNC_ERROR_MAX_CHARS, Repository
+from secondshift.workspaces.models import Workspace
+
+logger = logging.getLogger(__name__)
+
+
+class LocalRepositoryError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class LocalRepositoryLocation:
+    root: Path
+    # The branch SecondShift follows, and its newest commit.
+    branch: str
+    revision: str
+    repository_name: str
+    clone_url: str = ""
+
+
+# Git file modes for a regular and an executable file. Symbolic links and
+# submodules are refused so a path cannot point outside the repository.
+FILE_MODES = {"100644", "100755"}
+
+
+def git(root: Path, *arguments: str) -> str:
+    try:
+        completed = subprocess.run(
+            ["git", "-C", str(root), *arguments],
+            capture_output=True,
+            check=False,
+            text=True,
+            timeout=10,
+        )
+    except subprocess.TimeoutExpired as error:
+        raise LocalRepositoryError("Git took too long to answer.") from error
+    if completed.returncode:
+        # Git's own output can name folders on this machine, so it stays in
+        # the server log. Only owners may see local paths.
+        logger.info(
+            "git %s failed: %s",
+            arguments[0] if arguments else "",
+            completed.stderr.strip() or completed.stdout.strip(),
+        )
+        raise LocalRepositoryError("Git could not inspect the repository.")
+    return completed.stdout.strip()
+
+
+def committed_file(root: Path, revision: str, path: str) -> str:
+    """A file's text at a commit. Only regular files inside the repository."""
+    normalized = posixpath.normpath(path)
+    if normalized.startswith(("../", "/")) or normalized in ("..", "."):
+        raise LocalRepositoryError(f"{path} must be a file inside the repository.")
+    try:
+        entry = git(root, "ls-tree", revision, "--", normalized)
+    except LocalRepositoryError as error:
+        raise LocalRepositoryError(f"Git could not read {path}.") from error
+    if not entry:
+        raise LocalRepositoryError(f"{path} is not committed on the branch.")
+    mode = entry.split(maxsplit=1)[0]
+    if mode not in FILE_MODES:
+        raise LocalRepositoryError(f"{path} must be a regular file, not a link.")
+    try:
+        return git(root, "show", f"{revision}:{normalized}")
+    except LocalRepositoryError as error:
+        raise LocalRepositoryError(f"Git could not read {path}.") from error
+
+
+def current_branch(root: Path) -> str:
+    try:
+        return git(root, "symbolic-ref", "--short", "HEAD")
+    except LocalRepositoryError as error:
+        raise LocalRepositoryError(
+            "Check out the branch SecondShift should follow, then add the repository."
+        ) from error
+
+
+def branch_revision(root: Path, branch: str) -> str:
+    """The newest commit on a branch."""
+    try:
+        return git(root, "rev-parse", "--verify", f"refs/heads/{branch}^{{commit}}")
+    except LocalRepositoryError as error:
+        raise LocalRepositoryError(f"The branch {branch} does not exist.") from error
+
+
+def locate_local_repository(
+    raw_path: str, branch: str | None = None
+) -> LocalRepositoryLocation:
+    """Find the repository and the newest commit on a branch.
+
+    Without a branch it follows the one checked out now.
+    """
+    candidate = Path(raw_path).expanduser().resolve()
+    if not candidate.is_dir():
+        raise LocalRepositoryError("The selected path is not a directory.")
+
+    try:
+        root = Path(git(candidate, "rev-parse", "--show-toplevel")).resolve()
+    except LocalRepositoryError as error:
+        raise LocalRepositoryError(
+            "The selected path is not a Git repository."
+        ) from error
+
+    branch = branch or current_branch(root)
+    revision = branch_revision(root, branch)
+    try:
+        github_name = github_remote_name(git(root, "remote", "get-url", "origin"))
+    except LocalRepositoryError:
+        github_name = None
+    return LocalRepositoryLocation(
+        root=root,
+        branch=branch,
+        revision=revision,
+        repository_name=local_repository_name(root),
+        clone_url=f"{GITHUB_BASE_URL}/{github_name}" if github_name else "",
+    )
+
+
+def local_repository_name(root: Path) -> str:
+    digest = hashlib.sha256(str(root).encode()).hexdigest()[:8]
+    return f"local/{root.name}-{digest}"
+
+
+def save_local_location(
+    workspace: Workspace,
+    location: LocalRepositoryLocation,
+    config_error: str,
+    config_read: bool,
+) -> tuple[Repository, bool]:
+    """Save what was found in the folder.
+
+    Returns the repository and whether its GitHub origin changed.
+    """
+    previous = local_repository(workspace, str(location.root))
+    origin_changed = previous is not None and previous.clone_url != location.clone_url
+    synced = {"config_sha": location.revision} if config_read else {}
+    # Only a new or a removed repository is switched on. A resync keeps the
+    # owner's choice, as GitHub syncs do.
+    added = (
+        {"enabled": True, "removed_at": None}
+        if previous is None or previous.removed_at is not None
+        else {}
+    )
+    repository = save_local_repository(
+        workspace,
+        str(location.root),
+        {
+            "source_type": Repository.SourceType.LOCAL,
+            "full_name": location.repository_name,
+            "clone_url": location.clone_url,
+            "default_branch": location.branch,
+            "config_path": MANIFEST_FILE,
+            "checked_sha": location.revision,
+            "last_sync_error": config_error[:SYNC_ERROR_MAX_CHARS],
+            **added,
+            **synced,
+        },
+    )
+    return repository, origin_changed
+
+
+def local_directories(raw_path: str) -> tuple[Path, Path | None, list[Path]]:
+    requested = raw_path if raw_path.strip() else ""
+    try:
+        candidate = (
+            Path(requested).expanduser().resolve()
+            if requested
+            else Path.home().resolve()
+        )
+    except (OSError, RuntimeError, ValueError) as error:
+        target = "selected path" if requested else "home directory"
+        raise LocalRepositoryError(f"Unable to open the {target}.") from error
+    try:
+        is_directory = candidate.is_dir()
+    except (OSError, RuntimeError, ValueError) as error:
+        raise LocalRepositoryError(f"Unable to access {candidate}.") from error
+    if not is_directory:
+        raise LocalRepositoryError(f"{candidate} is not a directory.")
+    try:
+        entries = list(candidate.iterdir())
+        directories = sorted(
+            (item for item in entries if item.is_dir()),
+            key=lambda item: item.name.lower(),
+        )
+    except (OSError, RuntimeError, ValueError) as error:
+        raise LocalRepositoryError(f"Unable to read {candidate}.") from error
+    parent = None if candidate.parent == candidate else candidate.parent
+    return candidate, parent, directories

secondshift/repositories/management/commands/prune_source_cache.py

@@ -0,0 +1,15 @@
+import djclick as click
+
+from secondshift.repositories.sources import prune_source_cache
+
+
+@click.command()
+@click.option("--dry-run", is_flag=True, help="Show what would be removed.")
+def command(dry_run: bool) -> None:
+    """Remove cached repository sources no repository needs, and trim to size."""
+    pruned = prune_source_cache(dry_run=dry_run)
+    verb = "Would remove" if dry_run else "Removed"
+    click.echo(
+        f"{verb} {pruned.snapshots} snapshot(s) and {pruned.mirrors} mirror(s), "
+        f"{pruned.bytes / (1024 * 1024):.1f} MiB"
+    )

secondshift/repositories/migrations/0001_initial.py

@@ -0,0 +1,45 @@
+# Generated by Django 6.0.9 on 2026-10-26 14:22
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('connections', '0001_initial'),
+        ('workspaces', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Repository',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('source_type', models.CharField(choices=[('github', 'GitHub'), ('local', 'Local')], default='github', max_length=16)),
+                ('local_path', models.CharField(blank=True, max_length=1000)),
+                ('external_id', models.CharField(blank=True, max_length=64)),
+                ('full_name', models.CharField(max_length=255)),
+                ('clone_url', models.URLField(blank=True)),
+                ('default_branch', models.CharField(default='main', max_length=255)),
+                ('installation_id', models.CharField(blank=True, max_length=64)),
+                ('enabled', models.BooleanField(default=False)),
+                ('auto_sync', models.BooleanField(default=True)),
+                ('config_path', models.CharField(default='.secondshift.yaml', max_length=255)),
+                ('config_sha', models.CharField(blank=True, max_length=64)),
+                ('last_synced_at', models.DateTimeField(blank=True, null=True)),
+                ('removed_at', models.DateTimeField(blank=True, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('github_connection', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='repositories', to='connections.connection')),
+                ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='repositories', to='workspaces.workspace')),
+            ],
+            options={
+                'verbose_name_plural': 'repositories',
+                'ordering': ['full_name'],
+                'constraints': [models.UniqueConstraint(fields=('workspace', 'full_name'), name='unique_workspace_repository'), models.UniqueConstraint(condition=models.Q(('local_path', ''), _negated=True), fields=('workspace', 'local_path'), name='unique_workspace_local_repository')],
+            },
+        ),
+    ]

secondshift/repositories/migrations/0002_repository_last_sync_error.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.9 on 2026-10-26 15:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('repositories', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='repository',
+            name='last_sync_error',
+            field=models.CharField(blank=True, max_length=500),
+        ),
+    ]

secondshift/repositories/migrations/0003_repository_checked_sha.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.9 on 2026-11-03 13:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('repositories', '0002_repository_last_sync_error'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='repository',
+            name='checked_sha',
+            field=models.CharField(blank=True, max_length=64),
+        ),
+    ]

secondshift/repositories/migrations/0004_pull_request_settings.py

@@ -0,0 +1,23 @@
+# Generated by Django 6.0.9 on 2026-11-03 20:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('repositories', '0003_repository_checked_sha'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='repository',
+            name='allow_pull_requests',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='repository',
+            name='pull_request_ci',
+            field=models.BooleanField(default=False),
+        ),
+    ]

secondshift/repositories/migrations/0005_remove_repository_installation_id.py

@@ -0,0 +1,17 @@
+# Generated by Django 6.0.9 on 2026-11-03 23:46
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('repositories', '0004_pull_request_settings'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='repository',
+            name='installation_id',
+        ),
+    ]

secondshift/repositories/migrations/0006_allow_issues.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.9 on 2026-11-04 09:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('repositories', '0005_remove_repository_installation_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='repository',
+            name='allow_issues',
+            field=models.BooleanField(default=False),
+        ),
+    ]