quoll-cli 0.2.0__py3-none-any.whl

quoll/platforms/gitlab/api.py

@@ -0,0 +1,158 @@
+"""GitLab REST API (gitlab.com or a self-managed instance): projects, CI/CD variables, protected branches."""
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Optional, Sequence
+from urllib.parse import quote
+
+import httpx
+
+from quoll.config.settings import settings
+from quoll.platforms.base import PlatformAPI, PlatformError, Repo
+
+_ERROR_HINTS = {
+    401: "The token is invalid or expired (QUOLL_GITLAB_TOKEN, `glab auth login`, or ~/.config/quoll/config.toml).",
+    403: "The token lacks permissions: it needs the 'api' scope and the Maintainer role on the project.",
+    404: "Project or group not found (or not visible to this token).",
+}
+# Identifies the pipeline schedule quoll manages among the project's others.
+SCHEDULE_DESCRIPTION = "quoll: security scans"
+# Access levels: https://docs.gitlab.com/api/protected_branches/
+_NO_ONE = 0
+_MAINTAINER = 40
+
+
+class GitLabError(PlatformError):
+    pass
+
+
+def gitlab_client(base_url: Optional[str] = None) -> httpx.Client:
+    return httpx.Client(
+        base_url=f"{(base_url or settings.gitlab_url).rstrip('/')}/api/v4",
+        headers={"PRIVATE-TOKEN": settings.gitlab_token},
+        timeout=30,
+    )
+
+
+class GitLabAPI(PlatformAPI):
+    def __init__(self, base_url: Optional[str] = None) -> None:
+        self.base_url = base_url
+
+    def create_repo(self, name: str, private: bool = True) -> Repo:
+        with gitlab_client(self.base_url) as client:
+            payload: Dict[str, object] = {
+                "name": name,
+                "path": name,
+                "visibility": "private" if private else "public",
+                "description": "Scaffolded by quoll",
+                "initialize_with_readme": False,
+            }
+            if settings.gitlab_group:
+                response = client.get(f"/groups/{_id(settings.gitlab_group)}")
+                _raise_for_status(response, f"find the group {settings.gitlab_group}")
+                payload["namespace_id"] = response.json()["id"]
+            response = client.post("/projects", json=payload)
+            _raise_for_status(response, "create the project")
+            data = response.json()
+            return Repo(
+                path=data["path_with_namespace"], clone_url=data["http_url_to_repo"], web_url=data["web_url"]
+            )
+
+    def set_secrets(self, repo: str, secrets: Dict[str, str]) -> None:
+        """Create or update CI/CD variables, masked whenever GitLab accepts to mask the value."""
+        repo = self._full_path(repo)
+        with gitlab_client(self.base_url) as client:
+            for key, value in secrets.items():
+                payload = {"key": key, "value": value, "masked": _maskable(value), "protected": False}
+                response = client.put(f"/projects/{_id(repo)}/variables/{key}", json=payload)
+                if response.status_code == 404:
+                    response = client.post(f"/projects/{_id(repo)}/variables", json=payload)
+                _raise_for_status(response, f"save the variable {key}")
+
+    def ensure_schedule(self, repo: str, branch: str, cron: str) -> bool:
+        """The pipeline schedule quoll owns (found by its description), created or updated."""
+        repo = self._full_path(repo)
+        payload = {"description": SCHEDULE_DESCRIPTION, "ref": branch, "cron": cron, "cron_timezone": "UTC",
+                   "active": True}
+        schedules = f"/projects/{_id(repo)}/pipeline_schedules"
+        with gitlab_client(self.base_url) as client:
+            response = client.get(schedules, params={"per_page": 100})
+            _raise_for_status(response, "list the pipeline schedules")
+            existing = next((s for s in response.json() if s.get("description") == SCHEDULE_DESCRIPTION), None)
+            if existing:
+                response = client.put(f"{schedules}/{existing['id']}", json=payload)
+            else:
+                response = client.post(schedules, json=payload)
+            _raise_for_status(response, "save the pipeline schedule")
+        return True
+
+    def protect_branch(
+        self, repo: str, branch: str, required_checks: Sequence[str] = (), required_reviews: int = 0
+    ) -> List[str]:
+        """No direct pushes, merge requests only once the pipeline passed, linear history."""
+        repo = self._full_path(repo)
+        warnings: List[str] = []
+        project = f"/projects/{_id(repo)}"
+        with gitlab_client(self.base_url) as client:
+            # Protection settings can't be changed in place: unprotect, then protect again.
+            client.delete(f"{project}/protected_branches/{quote(branch, safe='')}")
+            response = client.post(
+                f"{project}/protected_branches",
+                json={
+                    "name": branch,
+                    "push_access_level": _NO_ONE,
+                    "merge_access_level": _MAINTAINER,
+                    "allow_force_push": False,
+                },
+            )
+            _raise_for_status(response, f"protect the branch {branch}")
+            response = client.put(
+                project,
+                json={
+                    "only_allow_merge_if_pipeline_succeeds": True,
+                    "only_allow_merge_if_all_discussions_are_resolved": True,
+                    "merge_method": "ff",
+                },
+            )
+            _raise_for_status(response, "require a successful pipeline to merge")
+            if required_reviews:
+                # Approval rules are a GitLab Premium feature.
+                response = client.post(
+                    f"{project}/approval_rules",
+                    json={"name": "quoll", "approvals_required": required_reviews},
+                )
+                if response.is_error:
+                    warnings.append(
+                        f"required approvals not set (GitLab {response.status_code}): they need GitLab Premium"
+                    )
+        return warnings
+
+    def _full_path(self, repo: str) -> str:
+        """group/project; a bare name belongs to the configured group, or to the authenticated user."""
+        if "/" in repo:
+            return repo
+        if settings.gitlab_group:
+            return f"{settings.gitlab_group}/{repo}"
+        with gitlab_client(self.base_url) as client:
+            response = client.get("/user")
+            _raise_for_status(response, "read the authenticated user")
+            return f"{response.json()['username']}/{repo}"
+
+
+def _id(path: str) -> str:
+    """Projects and groups are addressed by their URL-encoded full path."""
+    return quote(path, safe="")
+
+
+def _maskable(value: str) -> bool:
+    # https://docs.gitlab.com/ci/variables/#mask-a-cicd-variable
+    return len(value) >= 8 and bool(re.fullmatch(r"[A-Za-z0-9+/=@:.~_-]+", value))
+
+
+def _raise_for_status(response: httpx.Response, action: str) -> None:
+    if response.is_error:
+        hint = _ERROR_HINTS.get(response.status_code, "")
+        msg = f"GitLab {response.status_code} while trying to {action}."
+        if hint:
+            msg += f" {hint}"
+        raise GitLabError(msg)

quoll/platforms/gitlab/cd.py

@@ -0,0 +1,226 @@
+"""Turn a CDConfig into .gitlab/cd.yml: build, scan with Trivy, then push.
+
+.gitlab-ci.yml (`quoll ci`) includes this file. Its job runs in the `publish` stage, so only once every CI job
+of the pipeline passed.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from quoll.cd.config import CDConfig
+from quoll.ci.tools import SBOM_FILE, TRIVY_IMAGE, image_sbom
+from quoll.files import PlannedFile, header, planned
+from quoll.k8s.ci import HELM_IMAGE
+from quoll.k8s.config import Environment
+from quoll.k8s.deploy import DIGEST, IMAGE, helm_upgrade, url
+from quoll.platforms.gitlab.ci import CD_PATH, SCHEDULED
+from quoll.platforms.gitlab.pipeline import branch_rule, docker_job, dump, registry_cache
+from quoll.workspace import Delivery
+
+PIPELINE_PATH = Path(CD_PATH)
+
+# Same tags as docker/metadata-action on GitHub: branch, sha-<sha>, latest on the default branch,
+# 1.2.3 and 1.2 for a v1.2.3 tag (plus latest unless it is a pre-release).
+_TAGS = """tags="sha-$CI_COMMIT_SHORT_SHA"
+if [ -n "$CI_COMMIT_TAG" ]; then
+  version="${CI_COMMIT_TAG#v}"
+  case "$version" in
+    *-*) tags="$tags $version" ;;
+    *) tags="$tags $version ${version%.*} latest" ;;
+  esac
+else
+  tags="$tags $CI_COMMIT_REF_SLUG"
+  if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then tags="$tags latest"; fi
+fi
+tag_args=""
+for tag in $tags; do tag_args="$tag_args --tag $IMAGE:$tag"; done"""
+
+# The scanner runs as a container of the Docker-in-Docker daemon, next to the image it scans.
+_SCAN = (
+    "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock {trivy} image --no-progress --exit-code 1 "
+    '--severity {severity}{unfixed} "scan:$CI_COMMIT_SHA"'
+)
+
+# The digest buildx pushed, from its metadata file.
+_DIGEST = """DIGEST=$(sed -n 's/.*"containerimage.digest": *"\\([^"]*\\)".*/\\1/p' metadata.json | head -n 1)
+echo "Digest: $DIGEST\""""
+# Variables the publish job hands to the deploy jobs.
+DEPLOY_ENV = "deploy.env"
+# `release:` runs through glab.
+GLAB_IMAGE = "registry.gitlab.com/gitlab-org/cli:v1.130.0"
+
+_LOGIN = {
+    "gitlab": 'echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin '
+    '"$CI_REGISTRY"',
+    "dockerhub": 'echo "$DOCKERHUB_TOKEN" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin',
+}
+
+
+def render_pipeline(target: Union[CDConfig, List[Delivery]]) -> str:
+    """The delivery of a project, or of each workspace member with a [cd] (when its directory changed)."""
+    deliveries = [Delivery("", ".", target)] if isinstance(target, CDConfig) else target
+    # Not "image": a reserved top-level keyword.
+    doc: Dict[str, Any] = {_publish_job(d): _publish(d) for d in deliveries}
+    for d in deliveries:
+        for environment, settings in (d.k8s.environments.items() if d.k8s else ()):
+            job = f"{d.name}-deploy-{environment}" if d.name else f"deploy-{environment}"
+            doc[job] = _deploy(d, environment, settings)
+    released = [d for d in deliveries if d.cfg.sbom and d.cfg.release_tags]
+    if released:
+        doc["release"] = _release(released)
+    return header("cd") + dump(doc)
+
+
+def _publish(delivery: Delivery) -> Dict[str, Any]:
+    cfg, name, directory = delivery.cfg, delivery.name, delivery.directory
+    branches = [branch_rule(b) for b in cfg.branches]
+    if name and directory != ".":
+        # Branch pushes only publish what changed (tag pipelines count everything as changed).
+        changes = {"paths": [f"{directory}/**/*", "quoll.toml", ".gitlab-ci.yml", ".gitlab/**/*"]}
+        branches = [{**rule, "changes": changes} for rule in branches]
+    # Never from a scheduled pipeline (quoll ci's security scans run on the default branch).
+    rules: List[Dict[str, Any]] = [{"if": SCHEDULED, "when": "never"}, *branches]
+    if cfg.release_tags:
+        rules.append({"if": "$CI_COMMIT_TAG =~ /^v/"})
+
+    script = [_TAGS]
+    if any(not p.endswith("amd64") for p in cfg.platforms):
+        script.append("docker run --privileged --rm tonistiigi/binfmt --install all")
+    # A container builder (multi-platform images, provenance and SBOM attestations), with the layers the CI's docker
+    # job cached in the project's registry.
+    script.append(registry_cache(name or "ci"))
+    if cfg.scan:
+        script += [
+            # Built and loaded locally first: a vulnerable image never reaches the registry.
+            f'docker buildx build --platform {cfg.platforms[0]} --load $cache --tag "scan:$CI_COMMIT_SHA" .',
+            _SCAN.format(
+                trivy=TRIVY_IMAGE,
+                severity=",".join(cfg.severity),
+                unfixed=" --ignore-unfixed" if cfg.ignore_unfixed else "",
+            ),
+        ]
+    gitlab = cfg.registry == "gitlab"
+    script += [
+        _LOGIN["gitlab" if gitlab else "dockerhub"],
+        # Supply-chain metadata stored next to the image; the metadata file gives the pushed digest.
+        f"docker buildx build --platform {','.join(cfg.platforms)} --push $cache --provenance=mode=max --sbom=true "
+        "--metadata-file metadata.json $tag_args .",
+        _DIGEST,
+    ]
+    if cfg.sbom:
+        user, password = ("$CI_REGISTRY_USER", "$CI_REGISTRY_PASSWORD") if gitlab else (
+            "$DOCKERHUB_USERNAME", "$DOCKERHUB_TOKEN"
+        )
+        script.append(f'TRIVY_USERNAME="{user}" TRIVY_PASSWORD="{password}" {image_sbom()}')
+    if cfg.sign:
+        script += [
+            "apk add --no-cache cosign",
+            # Keyless: the job's OIDC token (id_tokens below) is the identity, recorded in Sigstore's public log.
+            'cosign sign --yes "$IMAGE@$DIGEST"',
+        ]
+        if cfg.sbom:
+            script.append(f'cosign attest --yes --type cyclonedx --predicate {SBOM_FILE} "$IMAGE@$DIGEST"')
+    script.append('echo "Published:"; for tag in $tags; do echo "  $IMAGE:$tag"; done; echo "  $IMAGE@$DIGEST"')
+    if delivery.k8s and delivery.k8s.environments:
+        # For the deploy jobs (a dotenv report becomes their variables).
+        script.append(f'echo "{DIGEST}=$DIGEST" > {DEPLOY_ENV}')
+
+    publish = docker_job(
+        "publish",
+        script,
+        timeout=30,
+        directory=directory,
+        rules=rules,
+        variables={"IMAGE": cfg.image},
+        # Never cancel a publication halfway, and never run two at once.
+        interruptible=False,
+        resource_group=f"publish-{name}" if name else "publish",
+    )
+    if cfg.sign:
+        publish["id_tokens"] = {"SIGSTORE_ID_TOKEN": {"aud": "sigstore"}}
+    if cfg.sbom:
+        sbom = _sbom_path(delivery)
+        publish["artifacts"] = {"paths": [sbom], "reports": {"cyclonedx": [sbom]}, "expire_in": "1 week"}
+    if delivery.k8s and delivery.k8s.environments:
+        reports = publish.setdefault("artifacts", {"expire_in": "1 week"}).setdefault("reports", {})
+        reports["dotenv"] = DEPLOY_ENV if directory == "." else f"{directory}/{DEPLOY_ENV}"
+    return publish
+
+
+def _publish_job(delivery: Delivery) -> str:
+    return f"{delivery.name}-publish" if delivery.name else "publish"
+
+
+def _deploy(delivery: Delivery, environment: str, settings: Environment) -> Dict[str, Any]:
+    """Deploy the image just published, by digest, once its signature checks out. KUBECONFIG is a file variable
+    scoped to the environment: a cluster each."""
+    cfg, k8s, directory = delivery.cfg, delivery.k8s, delivery.directory
+    publish_rules = _publish(delivery)["rules"]
+    if settings.deploy_on == "branch":
+        # Where the image is published from a branch (and the project changed, in a workspace).
+        rules = [r for r in publish_rules if not r.get("if", "").startswith("$CI_COMMIT_TAG")]
+    else:
+        # Production: on v* tags, started by hand (protect the environment to choose who can).
+        rules = [{"if": SCHEDULED, "when": "never"}, {"if": "$CI_COMMIT_TAG =~ /^v/", "when": "manual"}]
+    script = [] if directory == "." else [f'cd "{directory}"']
+    if cfg.sign:
+        script += [
+            "apk add --no-cache cosign",
+            # Signed keylessly by this project's pipelines (the publish job).
+            f'cosign verify "${IMAGE}@${DIGEST}" --certificate-identity-regexp "^$CI_SERVER_URL/$CI_PROJECT_PATH//" '
+            '--certificate-oidc-issuer "$CI_SERVER_URL" > /dev/null',
+        ]
+    # Without one, helm would try a cluster on localhost.
+    script.append(f'[ -n "${{KUBECONFIG:-}}" ] || {{ echo "No KUBECONFIG file variable for the {environment} '
+                  'environment: add the cluster\'s kubeconfig (CI/CD variable, type File)."; exit 1; }')
+    script.append(helm_upgrade(k8s, environment, settings))
+    target: Dict[str, Any] = {"name": environment, "action": "start"}
+    address = url(settings, k8s.tls)
+    if address:
+        target["url"] = address
+    return {
+        "stage": "deploy",
+        "image": {"name": HELM_IMAGE, "entrypoint": [""]},
+        "needs": [{"job": _publish_job(delivery), "artifacts": True}],
+        "rules": rules,
+        "variables": {IMAGE: cfg.image},
+        "environment": target,
+        "resource_group": f"deploy-{environment}-{delivery.name or 'app'}",
+        "interruptible": False,
+        "timeout": "15 minutes",
+        "script": script,
+    }
+
+
+def _sbom_path(delivery: Delivery) -> str:
+    return SBOM_FILE if delivery.directory == "." else f"{delivery.directory}/{SBOM_FILE}"
+
+
+def _release(deliveries: List[Delivery]) -> Dict[str, Any]:
+    """On v* tags, a GitLab release linking to the images' SBOMs, kept with this job's artifacts."""
+    links = []
+    for d in deliveries:
+        url = f"$CI_PROJECT_URL/-/jobs/$CI_JOB_ID/artifacts/raw/{_sbom_path(d)}"
+        label = f"{d.name} SBOM (CycloneDX)" if d.name else "SBOM (CycloneDX)"
+        links.append({"name": label, "url": url, "link_type": "other"})
+    images = ", ".join(d.cfg.image for d in deliveries)
+    plural = "s" if len(deliveries) > 1 else ""
+    return {
+        "stage": "publish",
+        "image": GLAB_IMAGE,
+        "needs": [{"job": f"{d.name}-publish" if d.name else "publish", "artifacts": True} for d in deliveries],
+        "rules": [{"if": "$CI_COMMIT_TAG =~ /^v/"}],
+        "script": ['echo "Release $CI_COMMIT_TAG"'],
+        "artifacts": {"paths": [_sbom_path(d) for d in deliveries], "expire_in": "never"},
+        "release": {
+            "tag_name": "$CI_COMMIT_TAG",
+            "description": f"Image{plural} {images} ($CI_COMMIT_TAG), with {'their' if plural else 'its'} SBOM "
+            "(CycloneDX).",
+            "assets": {"links": links},
+        },
+    }
+
+
+def plan(project_dir: Path, target: Union[CDConfig, List[Delivery]]) -> List[PlannedFile]:
+    return [planned(project_dir, PIPELINE_PATH, render_pipeline(target))]