commitguardian 0.1.0__py3-none-any.whl

commitguard/github/pull_requests.py

@@ -0,0 +1,53 @@
+"""Which pull request webhook actions matter to CommitGuard.
+
+=====================================  ==========================================
+Action                                 What happens
+=====================================  ==========================================
+``opened``, ``synchronize``,           scan the complete PR commit set
+``reopened``                           (``head ^base``, trusted base policy)
+``edited`` with a base branch change   scan again (different base and policy)
+``closed`` and merged                  record the final state (audit); no scan
+``closed`` without merge               cancel queued scans; no scan
+anything else (labels, reviews...)     ignored: cannot change the commit set
+=====================================  ==========================================
+
+A ``synchronize`` scan always evaluates ``base..new head``, never only the
+newly pushed commits, so an authoritative result covers every commit the PR
+would merge.
+"""
+
+from enum import StrEnum
+
+from commitguard.github.events import PULL_REQUEST_SCAN_ACTIONS, PullRequestEvent
+from commitguard.security.hashing import sha256_hex
+
+
+class PullRequestDisposition(StrEnum):
+    SCAN = "scan"
+    RECORD_MERGE = "record_merge"
+    CLOSE = "close"
+    IGNORE = "ignore"
+
+
+def disposition(event: PullRequestEvent) -> PullRequestDisposition:
+    if event.action in PULL_REQUEST_SCAN_ACTIONS:
+        return PullRequestDisposition.SCAN
+    if event.action == "edited" and event.base_changed:
+        return PullRequestDisposition.SCAN
+    if event.action == "closed":
+        return PullRequestDisposition.RECORD_MERGE if event.merged else PullRequestDisposition.CLOSE
+    return PullRequestDisposition.IGNORE
+
+
+def group_key(number: int) -> str:
+    return f"pull_request:{int(number)}"
+
+
+def branch_group_key(ref: str) -> str:
+    """Scan group for pushes to one branch (hashed: ref names are untrusted text)."""
+    return f"push:{sha256_hex(ref.encode('utf-8'))[:24]}"
+
+
+def merge_group_key(head_sha: str) -> str:
+    """Scan group for one merge queue candidate commit (each merge group has its own SHA)."""
+    return f"merge_group:{head_sha}"

commitguard/github/queue.py

@@ -0,0 +1,47 @@
+"""Event queue abstraction between the webhook receiver and scan workers.
+
+The webhook handler persists a :class:`~commitguard.github.storage.ScanJob`
+first and then puts its ID on the queue, so the queue is only a wake-up
+signal: a full queue, a crashed worker or a restart loses nothing, because
+:meth:`~commitguard.github.storage.SqliteStateStore.recoverable_jobs` returns
+every queued or abandoned job again. That keeps the in-process implementation
+honest and lets Redis, RabbitMQ, SQS or Kafka implement the same protocol later.
+"""
+
+import queue
+from typing import Protocol
+
+DEFAULT_QUEUE_SIZE = 10_000
+
+
+class EventQueue(Protocol):
+    def put(self, job_id: str) -> bool:
+        """Enqueue without blocking. False if the queue is full (the job stays stored)."""
+        ...
+
+    def get(self, timeout: float) -> str | None:
+        """The next job ID, or None after ``timeout`` seconds."""
+        ...
+
+    def size(self) -> int: ...
+
+
+class InProcessEventQueue:
+    def __init__(self, maxsize: int = DEFAULT_QUEUE_SIZE) -> None:
+        self._queue: queue.Queue[str] = queue.Queue(maxsize=maxsize)
+
+    def put(self, job_id: str) -> bool:
+        try:
+            self._queue.put_nowait(job_id)
+        except queue.Full:
+            return False
+        return True
+
+    def get(self, timeout: float) -> str | None:
+        try:
+            return self._queue.get(timeout=timeout)
+        except queue.Empty:
+            return None
+
+    def size(self) -> int:
+        return self._queue.qsize()

commitguard/github/recovery.py

@@ -0,0 +1,124 @@
+"""Operational recovery, run by the maintenance loop.
+
+What fails, and what happens next:
+
+=================================  =========================================================
+Failure                            Recovery
+=================================  =========================================================
+Event processing crashed           The event record is ``failed`` (or ``processing`` for
+(database error, process killed)   longer than 10 minutes and then marked ``failed``);
+                                   GitHub's redelivery of the same delivery ID is processed
+                                   again instead of being dropped as a duplicate.
+Scan could not complete: GitHub    The execution is ``error`` and its check failed closed
+API, network or fetch timeout      when it could be published. Up to
+                                   :data:`MAX_AUTOMATIC_RETRIES` new executions (trigger
+                                   ``retry``) are scheduled after 5 and 20 minutes, only
+                                   while the scan is still the newest for its pull request,
+                                   branch or merge group, the installation is active and
+                                   monitoring is on. A retry never publishes success for
+                                   commits it did not scan.
+Scan crashed on every attempt      The execution becomes ``error`` after three claims, is
+                                   audited, and its check is completed as a failure.
+Notification delivery failed       Handled by :mod:`commitguard.notifications.retry`
+                                   (bounded retries with backoff).
+=================================  =========================================================
+
+Authorization and configuration errors are not retried automatically: they
+need a human (fix permissions or configuration, then re-run the check).
+"""
+
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+from commitguard.audit.models import AuditEventType
+from commitguard.github.storage import JobState, ScanTrigger, SqliteStateStore
+from commitguard.observability.logging import correlation, get_logger
+from commitguard.observability.metrics import SCAN_RETRIES, Metrics
+from commitguard.services.audit import AuditService
+
+log = get_logger(__name__)
+
+MAX_AUTOMATIC_RETRIES = 2
+RETRY_BACKOFF = (timedelta(minutes=5), timedelta(minutes=20))
+RETRY_WINDOW = timedelta(hours=24)
+RETRYABLE_FAILURES = ("infrastructure", "timeout")
+RECOVERY_BATCH = 50
+
+
+class RecoveryService:
+    def __init__(
+        self,
+        store: SqliteStateStore,
+        audit: AuditService,
+        metrics: Metrics,
+        *,
+        enqueue: Callable[[str], bool],
+        now: Callable[[], datetime] = lambda: datetime.now(UTC),
+    ) -> None:
+        self._store = store
+        self._audit = audit
+        self._metrics = metrics
+        self._enqueue = enqueue
+        self._now = now
+
+    def run_once(self) -> dict[str, int]:
+        now = self._now()
+        stuck = self._store.fail_stuck_deliveries(now)
+        if stuck:
+            log.warning("webhook_deliveries_abandoned", count=stuck)
+        return {"stuck_deliveries": stuck, "scan_retries": self.retry_failed_scans()}
+
+    def retry_failed_scans(self) -> int:
+        now = self._now()
+        rows = self._store.query(
+            "SELECT j.job_id FROM scan_jobs j WHERE j.state = 'error' "
+            "AND j.failure_kind IN (?, ?) AND j.completed_at >= ? "
+            # still the newest execution of the newest scan of its group
+            "AND NOT EXISTS (SELECT 1 FROM scan_jobs n WHERE n.installation_id = j.installation_id "
+            "AND n.repository_id = j.repository_id AND n.group_key = j.group_key "
+            "AND n.sequence > j.sequence) "
+            "AND EXISTS (SELECT 1 FROM installations i WHERE i.installation_id = j.installation_id "
+            "AND i.state = 'active') "
+            "ORDER BY j.sequence LIMIT ?",
+            (*RETRYABLE_FAILURES, (now - RETRY_WINDOW).timestamp(), RECOVERY_BATCH),
+        )
+        scheduled = 0
+        for row in rows:
+            job = self._store.get_job(str(row["job_id"]))
+            if job is None or job.state is not JobState.ERROR or job.completed_at is None:
+                continue
+            retries = sum(
+                1
+                for e in self._store.list_executions(
+                    job.installation_id, job.repository.id, job.scan_key
+                )
+                if e.trigger is ScanTrigger.RETRY
+            )
+            if retries >= MAX_AUTOMATIC_RETRIES:
+                continue
+            if now - job.completed_at < RETRY_BACKOFF[retries]:
+                continue
+            if not self._store.monitoring_enabled(job.installation_id, job.repository.id):
+                continue
+            execution, created = self._store.create_execution(
+                job, trigger=ScanTrigger.RETRY, now=now
+            )
+            if not created:
+                continue
+            with correlation(job_id=execution.job_id, repository=job.repository.full_name):
+                self._audit.record(
+                    AuditEventType.SCAN_RETRY_SCHEDULED,
+                    installation_id=job.installation_id,
+                    repository_id=job.repository.id,
+                    repository=job.repository.full_name,
+                    head_sha=job.head_sha,
+                    job=execution.job_id,
+                    previous_scan=job.job_id,
+                    execution=execution.execution,
+                    failure_kind=job.failure_kind,
+                    retry=retries + 1,
+                )
+            self._metrics.increment(SCAN_RETRIES)
+            self._enqueue(execution.job_id)
+            scheduled += 1
+        return scheduled

commitguard/github/repositories.py

@@ -0,0 +1,305 @@
+"""Metadata-only repository mirrors for server-side scanning.
+
+The GitHub App needs the same commit objects the GitHub Action and the Git
+hooks analyse, but it must never check out, build or run repository code. Each
+repository gets a *bare, partial* mirror under the App's data directory::
+
+    <data_dir>/mirrors/<installation_id>/<repository_id>.git
+
+* paths use numeric IDs only (no repository names: no traversal, no renames);
+* ``git init --bare --template=``: no hooks, no work tree, nothing checked out;
+* ``--filter=blob:none``: commits and trees are fetched, **file contents are
+  not** - only the CommitGuard configuration blobs at the commits being
+  evaluated are fetched explicitly by object ID;
+* the exact SHAs from the (verified) event are fetched, never branch names
+  chosen by a payload; the default branch comes from the GitHub API;
+* hardening per fetch: ``protocol.allow=never`` except the allowed protocol,
+  no redirects, no credential helpers, no submodules, no tags, hooks path
+  pointing at the null device, automatic GC and maintenance off;
+* the installation token is passed as an HTTP header through ``GIT_CONFIG_*``
+  environment variables (not in the URL, the command line or the mirror's
+  config) and is registered for redaction;
+* every later Git read uses ``GIT_NO_LAZY_FETCH=1``, so analysis never touches
+  the network.
+"""
+
+import base64
+import os
+import shutil
+import stat
+import threading
+import time
+from collections.abc import Callable, Iterable, Sequence
+from pathlib import Path
+from typing import Protocol
+from urllib.parse import quote, urlsplit
+
+from commitguard.config.defaults import CONFIG_FILENAMES
+from commitguard.exceptions.git import GitCommandError, GitError
+from commitguard.exceptions.service import InfrastructureError, ScanError
+from commitguard.git.commands import git_version, run_git
+from commitguard.git.repository import Repository
+from commitguard.github.errors import safe_text
+from commitguard.github.identifiers import RepositoryRef
+from commitguard.security.secrets import Secret, register_secret
+from commitguard.security.validation import is_git_sha, validate_repository_path
+
+MIRROR_GIT_ENV = {"GIT_NO_LAZY_FETCH": "1"}
+# GIT_NO_LAZY_FETCH (Git 2.45) is what keeps analysis of a partial mirror offline;
+# older Git silently ignores it, so the App refuses to run with it.
+MINIMUM_MIRROR_GIT_VERSION = (2, 45)
+DEFAULT_FETCH_TIMEOUT_SECONDS = 600.0
+LAST_USED_MARKER = "commitguard-last-used"
+_REF_NAME_MAX = 255
+
+
+def _clear_read_only_and_retry(
+    function: Callable[[str], object], path: str, _error: BaseException
+) -> None:
+    """Git writes pack files read-only; on Windows they cannot be deleted until writable."""
+    os.chmod(path, stat.S_IWRITE)
+    function(path)
+
+
+class FetchTimeoutError(InfrastructureError):
+    """Fetching commits from GitHub took longer than the configured limit."""
+
+
+class RemoteLocator(Protocol):
+    def url_for(self, repository: RepositoryRef) -> str: ...
+
+
+class GitHubRemoteLocator:
+    """``https://github.com/<owner>/<name>.git`` from a validated repository reference."""
+
+    def __init__(self, base_url: str = "https://github.com/") -> None:
+        parts = urlsplit(base_url)
+        if parts.scheme != "https" or not parts.hostname or parts.username or parts.query:
+            raise ValueError("GitHub Git base URL must be an https URL without credentials")
+        self._base = base_url.rstrip("/") + "/"
+
+    def url_for(self, repository: RepositoryRef) -> str:
+        return (
+            f"{self._base}{quote(repository.owner, safe='')}/{quote(repository.name, safe='')}.git"
+        )
+
+
+def _auth_environment(url: str, token: Secret) -> dict[str, str]:
+    parts = urlsplit(url)
+    prefix = f"{parts.scheme}://{parts.netloc}/"
+    basic = base64.b64encode(f"x-access-token:{token.reveal()}".encode()).decode("ascii")
+    register_secret(basic)
+    return {
+        "GIT_CONFIG_COUNT": "1",
+        "GIT_CONFIG_KEY_0": f"http.{prefix}.extraHeader",
+        "GIT_CONFIG_VALUE_0": f"Authorization: Basic {basic}",
+    }
+
+
+def _valid_branch(name: str) -> bool:
+    return (
+        0 < len(name) <= _REF_NAME_MAX
+        and not name.startswith(("-", "/"))
+        and name.isprintable()
+        and " " not in name
+        and run_git(["check-ref-format", f"refs/heads/{name}"], check=False).ok
+    )
+
+
+def require_mirror_git() -> None:
+    try:
+        version = git_version()
+    except GitError:
+        raise InfrastructureError("git is not available") from None
+    if version[:2] < MINIMUM_MIRROR_GIT_VERSION:
+        wanted = ".".join(map(str, MINIMUM_MIRROR_GIT_VERSION))
+        found = ".".join(map(str, version))
+        raise InfrastructureError(f"the GitHub App needs Git {wanted} or newer (found {found})")
+
+
+class MirrorManager:
+    def __init__(
+        self,
+        root: Path,
+        locator: RemoteLocator,
+        *,
+        allowed_protocols: Sequence[str] = ("https",),
+        fetch_timeout: float = DEFAULT_FETCH_TIMEOUT_SECONDS,
+    ) -> None:
+        require_mirror_git()
+        self.root = root
+        self._locator = locator
+        self._protocols = tuple(allowed_protocols)
+        self._fetch_timeout = fetch_timeout
+        self._locks: dict[tuple[int, int], threading.Lock] = {}
+        self._locks_guard = threading.Lock()
+
+    def path_for(self, installation_id: int, repository_id: int) -> Path:
+        return self.root / str(int(installation_id)) / f"{int(repository_id)}.git"
+
+    def _lock(self, installation_id: int, repository_id: int) -> threading.Lock:
+        with self._locks_guard:
+            return self._locks.setdefault((installation_id, repository_id), threading.Lock())
+
+    def _git_options(self) -> list[str]:
+        options = ["-c", "protocol.allow=never"]
+        for protocol in self._protocols:
+            options += ["-c", f"protocol.{protocol}.allow=always"]
+        options += [
+            "-c", "http.followRedirects=false",
+            "-c", "credential.helper=",
+            "-c", f"core.hooksPath={os.devnull}",
+            "-c", "submodule.recurse=false",
+            "-c", "fetch.recurseSubmodules=false",
+            "-c", "fetch.writeCommitGraph=false",
+            "-c", "gc.auto=0",
+            "-c", "maintenance.auto=false",
+        ]  # fmt: skip
+        return options
+
+    def _init(self, path: Path, url: str) -> None:
+        if not (path / "HEAD").is_file():
+            path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+            run_git(["init", "--quiet", "--bare", "--template=", "--", str(path)])
+        for key, value in (
+            ("remote.origin.url", url),
+            ("remote.origin.promisor", "true"),
+            ("remote.origin.partialclonefilter", "blob:none"),
+            ("core.hooksPath", os.devnull),
+            ("gc.auto", "0"),
+            ("maintenance.auto", "false"),
+        ):
+            run_git(["config", "--end-of-options", key, value], cwd=path)
+
+    def _fetch(
+        self, path: Path, refspecs: Sequence[str], env: dict[str, str], *, check: bool = True
+    ) -> bool:
+        args = [
+            *self._git_options(),
+            "fetch",
+            "--quiet",
+            "--no-tags",
+            "--no-write-fetch-head",
+            "--no-recurse-submodules",
+            "--filter=blob:none",
+            "origin",
+            *refspecs,
+        ]
+        try:
+            result = run_git(
+                args, cwd=path, check=False, timeout=self._fetch_timeout, extra_env=env
+            )
+        except GitError as exc:
+            if "timed out" in str(exc):
+                raise FetchTimeoutError("fetching commits from GitHub timed out") from None
+            raise InfrastructureError(f"git fetch failed: {safe_text(str(exc))}") from None
+        if result.ok or not check:
+            return result.ok
+        lines = result.stderr.decode("utf-8", "replace").strip().splitlines()
+        detail = safe_text(lines[-1]) if lines else "no details"
+        if any(marker in detail for marker in ("not our ref", "couldn't find remote ref")):
+            raise ScanError(f"a commit to scan is not available on GitHub ({detail})")
+        raise InfrastructureError(f"could not fetch commits from GitHub ({detail})")
+
+    def prepare(
+        self,
+        installation_id: int,
+        repository: RepositoryRef,
+        token: Secret | None,
+        *,
+        required: Sequence[str],
+        optional: Sequence[str] = (),
+        branches: Sequence[str] = (),
+        config_path: str | None = None,
+    ) -> Repository:
+        """Fetch commits (and config blobs) into the mirror and return it for analysis."""
+        for oid in (*required, *optional):
+            if not is_git_sha(oid):
+                raise ScanError("mirror fetch requires full commit ids")
+        if config_path is not None:
+            validate_repository_path(config_path)
+        url = self._locator.url_for(repository)
+        env = dict(MIRROR_GIT_ENV)
+        if token is not None:
+            env.update(_auth_environment(url, token))
+        path = self.path_for(installation_id, repository.id)
+        with self._lock(installation_id, repository.id):
+            try:
+                self._init(path, url)
+            except GitCommandError as exc:
+                raise InfrastructureError(
+                    f"could not prepare mirror: {safe_text(str(exc))}"
+                ) from None
+            wanted = sorted(set(required))
+            if wanted:
+                self._fetch(path, [f"+{oid}:refs/commitguard/{oid}" for oid in wanted], env)
+            for oid in sorted(set(optional) - set(wanted)):
+                self._fetch(path, [f"+{oid}:refs/commitguard/{oid}"], env, check=False)
+            tips = []
+            for branch in branches:
+                if not _valid_branch(branch):
+                    continue
+                refspec = f"+refs/heads/{branch}:refs/remotes/origin/{branch}"
+                if self._fetch(path, [refspec], env, check=False):
+                    tips.append(f"refs/remotes/origin/{branch}")
+            blobs = self._missing_config_blobs(path, [*wanted, *optional, *tips], config_path)
+            if blobs:
+                self._fetch(path, blobs, env)
+            (path / LAST_USED_MARKER).touch()
+        return Repository(root=path, git_dir=path, git_env=MIRROR_GIT_ENV)
+
+    def _missing_config_blobs(
+        self, path: Path, revisions: Iterable[str], config_path: str | None
+    ) -> list[str]:
+        names = [config_path] if config_path else list(CONFIG_FILENAMES)
+        missing: set[str] = set()
+        for revision in revisions:
+            listed = run_git(
+                ["ls-tree", "-z", "--end-of-options", revision, "--", *names],
+                cwd=path,
+                check=False,
+                extra_env=MIRROR_GIT_ENV,
+            )
+            if not listed.ok:
+                continue  # optional commit that could not be fetched
+            for entry in filter(None, listed.stdout.split(b"\x00")):
+                meta = entry.partition(b"\t")[0].decode("ascii", "replace").split()
+                if len(meta) == 3 and meta[1] == "blob" and is_git_sha(meta[2]):
+                    exists = run_git(
+                        ["cat-file", "-e", meta[2]], cwd=path, check=False, extra_env=MIRROR_GIT_ENV
+                    )
+                    if not exists.ok:
+                        missing.add(meta[2])
+        return sorted(missing)
+
+    def _remove(self, target: Path) -> None:
+        root = self.root.resolve()
+        resolved = target.resolve()
+        if resolved == root or not resolved.is_relative_to(root) or target.is_symlink():
+            raise InfrastructureError("refusing to remove a path outside the mirror directory")
+        if resolved.exists():
+            shutil.rmtree(resolved, onexc=_clear_read_only_and_retry)
+
+    def remove_repository(self, installation_id: int, repository_id: int) -> None:
+        with self._lock(installation_id, repository_id):
+            self._remove(self.path_for(installation_id, repository_id))
+
+    def remove_installation(self, installation_id: int) -> None:
+        self._remove(self.root / str(int(installation_id)))
+
+    def purge_unused(self, older_than_seconds: float) -> int:
+        """Remove mirrors not used for ``older_than_seconds`` (retention)."""
+        if not self.root.is_dir():
+            return 0
+        cutoff = time.time() - older_than_seconds
+        removed = 0
+        for installation_dir in self.root.iterdir():
+            if not installation_dir.name.isdigit() or installation_dir.is_symlink():
+                continue
+            for mirror in installation_dir.glob("*.git"):
+                marker = mirror / LAST_USED_MARKER
+                used = marker.stat().st_mtime if marker.exists() else mirror.stat().st_mtime
+                if used < cutoff:
+                    self._remove(mirror)
+                    removed += 1
+        return removed

commitguard/github/server.py

@@ -0,0 +1,52 @@
+"""A small threaded HTTP server for development and single-host deployments.
+
+It serves the WSGI application from :mod:`commitguard.github.app` on a local
+address (default ``127.0.0.1``). It does not terminate TLS: expose it to
+GitHub only through a TLS-terminating reverse proxy or a development tunnel.
+Access logs are structured, contain no query strings or headers, and every
+connection has a socket timeout.
+"""
+
+from collections.abc import Callable, Iterable
+from socketserver import ThreadingMixIn
+from wsgiref.simple_server import WSGIRequestHandler, WSGIServer, make_server
+from wsgiref.types import StartResponse, WSGIEnvironment
+
+from commitguard.observability.logging import get_logger
+
+log = get_logger(__name__)
+
+SOCKET_TIMEOUT_SECONDS = 30
+
+
+class _ThreadingWSGIServer(ThreadingMixIn, WSGIServer):
+    daemon_threads = True
+    allow_reuse_address = True
+
+
+class _QuietHandler(WSGIRequestHandler):
+    timeout = SOCKET_TIMEOUT_SECONDS
+
+    def log_request(self, code: int | str = "-", size: int | str = "-") -> None:
+        log.info(
+            "http_request",
+            method=self.command,
+            path=(self.path or "").split("?", 1)[0][:200],
+            status=str(code),
+        )
+
+    def log_message(self, format: str, *args: object) -> None:
+        log.debug("http_server_message")
+
+
+def serve(
+    application: Callable[[WSGIEnvironment, StartResponse], Iterable[bytes]],
+    *,
+    host: str = "127.0.0.1",
+    port: int = 8080,
+) -> None:
+    with make_server(
+        host, port, application, server_class=_ThreadingWSGIServer, handler_class=_QuietHandler
+    ) as httpd:
+        log.info("http_server_listening", host=host, port=port)
+        httpd.serve_forever()