quoll-cli 0.2.0__py3-none-any.whl

quoll/commands/ci.py

@@ -0,0 +1,521 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Dict, List, Optional, Sequence, Tuple, Union
+
+import questionary
+import typer
+from pydantic import ValidationError
+from rich.table import Table
+
+from quoll import platforms
+from quoll.ci.config import CONFIG_FILE, EXTENSIBLE_JOBS, CIConfig, ConfigError, CustomStep, load_config, save_config
+from quoll.ci.detect import Detection, configured_platform, default_branch, default_config, detect
+from quoll.commands._common import (
+    DirOption,
+    ask,
+    check_files,
+    check_platform,
+    confirm_overwrite,
+    ensure_token,
+    platform_option,
+    resolve_dir,
+    supported_hint,
+    update_files,
+)
+from quoll.config import quoll_toml
+from quoll.console import console
+from quoll.files import eject as eject_text
+from quoll.files import write
+from quoll.platforms import PlatformError
+from quoll.platforms.base import PlatformAPI
+from quoll.workspace import Member, Repo, RepoCIConfig, discover, find_root, load_repo, member_name, save_workspace
+
+app = typer.Typer(help="Generate and maintain the CI pipeline (GitHub Actions or GitLab CI).", no_args_is_help=True)
+
+
+@app.command()
+def init(
+    project_dir: Path = DirOption,
+    yes: bool = typer.Option(False, "--yes", "-y", help="Accept detected defaults without asking."),
+    force: bool = typer.Option(False, "--force", help="Overwrite a pipeline that was not generated by quoll."),
+    platform: Optional[str] = platform_option(),
+    local: bool = typer.Option(
+        False, "--local", "--no-github", help="Only write files: skip branch protection and secrets."
+    ),
+) -> None:
+    """Detect the project (or the projects of a monorepo), write quoll.toml and generate the CI pipeline."""
+    project_dir = resolve_dir(project_dir)
+    check_platform(platform)
+    console.print(f"\n[bold]quoll ci[/bold] — {project_dir.name}\n")
+
+    if _configured(project_dir) and not force:
+        if yes or not ask(questionary.confirm(
+            f"{CONFIG_FILE} already configures CI. Detect again and replace it?", default=False
+        )):
+            console.print("Nothing changed. Use [bold]quoll ci update[/bold] to regenerate from quoll.toml.")
+            raise typer.Exit(0)
+
+    members = discover(project_dir)
+    if "." in members and len(members) > 1:
+        # The root's quoll.toml can't be both a project's and the workspace's.
+        others = ", ".join(m for m in members if m != ".")
+        console.print(f"[yellow]![/yellow] Also found {others}: a workspace needs each project in a directory of its "
+                      "own (move the root's into one). Setting up the root project alone.\n")
+    elif members and members != ["."]:  # the root alone: a single project
+        _init_workspace(project_dir, members, yes=yes, force=force, platform=platform, local=local)
+        return
+
+    det = detect(project_dir)
+    if det.stack is None:
+        console.print(f"[red]Error:[/red] no supported project found.\n{supported_hint()}")
+        raise typer.Exit(1)
+    if platform:
+        det.platform = platform
+    _print_detection(det)
+
+    cfg = default_config(det)
+    if not yes:
+        cfg = _customize(cfg, det)
+
+    files = _plan(project_dir, Repo.single(cfg))
+    if not confirm_overwrite(files, yes=yes, force=force):
+        raise typer.Exit(1)
+
+    save_config(project_dir, cfg)
+    write(project_dir, files)
+    console.print()
+    console.print(f"[green]✓[/green] {CONFIG_FILE}")
+    for f in files:
+        console.print(f"[green]✓[/green] {f.path}")
+
+    repo = Repo.single(cfg)
+    _finish(project_dir, repo, [cfg], det, files, yes=yes, local=local)
+
+
+def _init_workspace(
+    root: Path, found: List[str], *, yes: bool, force: bool, platform: Optional[str], local: bool
+) -> None:
+    """A monorepo: one quoll.toml per project, the workspace's at the root, one pipeline for all."""
+    remote = platforms.remote(root)
+    name = platform or configured_platform(root) or platforms.detect(root, remote)
+    table = Table(title="Projects found", box=None, padding=(0, 2), title_justify="left")
+    for column in ("Directory", "Stack", "Framework", "Tests"):
+        table.add_column(column, style="bold" if column == "Directory" else "")
+    detections = {d: detect(root / d) for d in found}
+    for directory, det in detections.items():
+        table.add_row(directory, det.stack or "-", det.framework or "-", det.test_runner or "-")
+    console.print(table)
+    console.print()
+    members = found if yes else ask(questionary.checkbox(
+        "Projects of the workspace", choices=[questionary.Choice(d, d, checked=True) for d in found]
+    ))
+    if not members:
+        raise typer.Exit(1)
+
+    configs: List[CIConfig] = []
+    for directory in members:
+        det = detections[directory]
+        det.platform = name
+        console.print(f"\n[bold]{directory}[/bold]")
+        _print_detection(det)
+        cfg = default_config(det)
+        if not yes:
+            cfg = _customize(cfg, det, workspace=True)
+        configs.append(cfg.in_directory(directory))
+
+    shared = RepoCIConfig(platform=name, branches=[default_branch(root)])
+    repo = Repo(
+        platform=name,
+        branches=list(shared.branches),
+        dependency_updates=shared.dependency_updates,
+        schedule=shared.schedule,
+        secrets=shared.security.secrets,
+        sast=shared.security.sast,
+        members=[Member(member_name(c.directory), c) for c in configs],
+    )
+    files = _plan(root, repo)
+    if not confirm_overwrite(files, yes=yes, force=force):
+        raise typer.Exit(1)
+    for cfg in configs:
+        save_config(root / cfg.directory, cfg)
+    save_workspace(root, members, shared)
+    write(root, files)
+    console.print()
+    console.print(f"[green]✓[/green] {CONFIG_FILE} (workspace: {', '.join(members)})")
+    for cfg in configs:
+        console.print(f"[green]✓[/green] {cfg.path(CONFIG_FILE)}")
+    for f in files:
+        console.print(f"[green]✓[/green] {f.path}")
+    det = detections[members[0]]
+    det.env_vars = {k: v for d in members for k, v in detections[d].env_vars.items()}
+    _finish(root, repo, configs, det, [], yes=yes, local=local, extra=[cfg.path(CONFIG_FILE) for cfg in configs])
+
+
+def _finish(
+    root: Path,
+    repo: Repo,
+    configs: List[CIConfig],
+    det: Detection,
+    files: list,
+    *,
+    yes: bool,
+    local: bool,
+    extra: Sequence[str] = (),
+) -> None:
+    platform = platforms.get(repo.platform)
+    if not yes and not local:
+        _remote_setup(repo, configs, det)
+    elif repo.schedule.enabled and platform.schedules_via_api:
+        console.print("\n[yellow]![/yellow] The weekly security scans need a pipeline schedule: run "
+                      "[bold]quoll ci schedule[/bold] once pushed.")
+
+    planned_files = files or _plan(root, repo)
+    paths = sorted({f.path.parts[0] + ("/" if len(f.path.parts) > 1 else "") for f in planned_files})
+    console.print("\n[bold green]Done![/bold green] Commit and push to run the pipeline:")
+    console.print(f"  git add {' '.join([CONFIG_FILE, *extra, *paths])}")
+    console.print('  git commit -m "ci: add quoll pipeline"')
+    console.print("  git push")
+    console.print(f"\n[dim]{platform.merge_gate()}[/dim]")
+
+
+@app.command()
+def update(
+    project_dir: Path = DirOption,
+    yes: bool = typer.Option(False, "--yes", "-y", help="Write changes without asking."),
+    force: bool = typer.Option(False, "--force", help="Overwrite a pipeline that was not generated by quoll."),
+) -> None:
+    """Regenerate the pipeline from quoll.toml (the workspace's, from any of its directories)."""
+    root, repo = _repo(resolve_dir(project_dir))
+    update_files(root, _plan(root, repo), yes=yes, force=force, what="CI is")
+
+
+@app.command()
+def check(project_dir: Path = DirOption) -> None:
+    """Fail if the pipeline differs from what quoll.toml describes (use it in CI or pre-commit)."""
+    root, repo = _repo(resolve_dir(project_dir))
+    check_files([f for f in _plan(root, repo) if not f.create_only], "ci")
+
+
+@app.command()
+def eject(
+    project_dir: Path = DirOption,
+    yes: bool = typer.Option(False, "--yes", "-y", help="Don't ask for confirmation."),
+) -> None:
+    """Stop managing the CI: its files become yours, as they are, and [ci] leaves quoll.toml."""
+    root, repo = _repo(resolve_dir(project_dir))
+    files = [f for f in _plan(root, repo) if not f.create_only and f.existing and f.managed]
+    if not yes and not ask(questionary.confirm(
+        f"quoll will no longer update {', '.join(str(f.path) for f in files)}. Eject?", default=False
+    )):
+        raise typer.Exit(1)
+    for f in files:
+        (root / f.path).write_text(eject_text(f.existing, "ci"))
+        console.print(f"[green]✓[/green] {f.path} is yours")
+    for directory in sorted({".", *(m.directory for m in repo.members)}):
+        quoll_toml.remove_section(root / directory, "ci")
+        console.print(f"[green]✓[/green] [ci] removed from {Path(directory, CONFIG_FILE).as_posix()}")
+
+
+@app.command()
+def schedule(project_dir: Path = DirOption) -> None:
+    """Create or update the schedule of the security scans ([ci.schedule]) where it lives outside the pipeline."""
+    root, repo = _repo(resolve_dir(project_dir))
+    platform = platforms.get(repo.platform)
+    if not repo.schedule.enabled:
+        console.print("Scheduled scans are off ([ci.schedule] enabled = false).")
+        return
+    if not platform.schedules_via_api:
+        console.print(f"[green]✓[/green] On {platform.label} the schedule ({repo.schedule.cron}) is in the pipeline "
+                      "itself: `quoll ci update` keeps it current.")
+        return
+    remote = platforms.remote(root)
+    if remote is None or platforms.hosting(remote) != platform.name:
+        console.print(f"[red]Error:[/red] no {platform.label} remote (origin) found.")
+        raise typer.Exit(1)
+    if not ensure_token(platform.name, remote.host):
+        raise typer.Exit(1)
+    _schedule(platform.api(remote.host), remote.path, repo.branches[0], repo.schedule.cron)
+
+
+@app.command(name="add-step")
+def add_step(
+    project_dir: Path = DirOption,
+    job: Optional[str] = typer.Option(None, help=f"Job to extend: {', '.join(EXTENSIBLE_JOBS)}."),
+    when: Optional[str] = typer.Option(None, help="before (once dependencies are installed) or after (at the end)."),
+    name: Optional[str] = typer.Option(None, help="Step name, shown in the CI."),
+    run: Optional[str] = typer.Option(None, help="Shell command, run in the project's directory."),
+    uses: Optional[str] = typer.Option(None, help="A GitHub Action instead (GitHub only), e.g. owner/action@v1."),
+    with_: List[str] = typer.Option([], "--with", help="Action input KEY=VALUE (repeatable)."),
+    secret: List[str] = typer.Option([], "--secret", help="CI secret / variable the step reads (repeatable)."),
+) -> None:
+    """Add a step to a generated job ([[ci.steps]] in quoll.toml), then regenerate the pipeline."""
+    project_dir = resolve_dir(project_dir)
+    if "workspace" in _read(project_dir):
+        members = load_repo(project_dir).members
+        console.print(f"Steps belong to a project of the workspace: run with -d <project> "
+                      f"({', '.join(m.cfg.directory for m in members)}).")
+        raise typer.Exit(1)
+    try:
+        cfg = load_config(project_dir)
+    except ConfigError as e:
+        console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)
+
+    job = job or ask(questionary.select("Job", choices=list(EXTENSIBLE_JOBS)))
+    when = when or ask(questionary.select("When", choices=[
+        questionary.Choice("before its own commands (dependencies installed)", "before"),
+        questionary.Choice("at the end", "after"),
+    ]))
+    name = name or ask(questionary.text("Step name", validate=lambda v: bool(v.strip()) or "Required"))
+    if not run and not uses:
+        run = ask(questionary.text("Command", validate=lambda v: bool(v.strip()) or "Required"))
+    if uses and cfg.platform != "github":
+        console.print("[red]Error:[/red] actions (--uses) are GitHub's: give a command with --run.")
+        raise typer.Exit(1)
+    inputs = dict(item.split("=", 1) for item in with_ if "=" in item)
+    try:
+        added = CustomStep.model_validate({
+            "job": job, "when": when, "name": name, "run": run or "", "uses": uses or "", "with": inputs,
+            "secrets": list(secret),
+        })
+    except ValidationError as e:
+        console.print(f"[red]Error:[/red] {e.errors()[0]['msg']}")
+        raise typer.Exit(1)
+    cfg.steps.append(added)
+    save_config(project_dir, cfg)
+    console.print(f"[green]✓[/green] {CONFIG_FILE}: step '{name}' added to the {job} job")
+    root, repo = _repo(project_dir)
+    files = [f for f in _plan(root, repo) if f.changed]
+    if any(not f.managed for f in files):
+        console.print("[yellow]![/yellow] The pipeline was written by hand: run [bold]quoll ci update --force[/bold] "
+                      "to regenerate it.")
+        raise typer.Exit(1)
+    write(root, files)
+    for f in files:
+        console.print(f"[green]✓[/green] {f.path} updated")
+
+
+def _read(project_dir: Path) -> dict:
+    try:
+        return quoll_toml.read(project_dir)
+    except ConfigError:
+        return {}
+
+
+@app.command(name="detect")
+def detect_cmd(project_dir: Path = DirOption) -> None:
+    """Show what quoll detects in the project (or each project of a monorepo), without writing anything."""
+    project_dir = resolve_dir(project_dir)
+    members = discover(project_dir)
+    if members and members != ["."]:
+        for directory in members:
+            console.print(f"[bold]{directory}[/bold]")
+            _print_detection(detect(project_dir / directory))
+        return
+    det = detect(project_dir)
+    if det.stack is None:
+        console.print(f"[yellow]No supported project found.[/yellow]\n{supported_hint()}")
+        raise typer.Exit(1)
+    _print_detection(det)
+
+
+# --------------------------------------------------------------------------- interactive
+
+
+def _customize(cfg: CIConfig, det: Detection, workspace: bool = False) -> CIConfig:
+    data = cfg.model_dump(by_alias=True)
+    # A workspace scans secrets and code once, for the whole repository (its root quoll.toml).
+    repository_scans = [] if workspace else [
+        questionary.Choice("Secret scanning (gitleaks)", "secrets", checked=True),
+        questionary.Choice("Static analysis (Semgrep)", "sast", checked=True),
+    ]
+    jobs = [
+        questionary.Choice("Lint", "lint", checked=cfg.lint.enabled),
+        questionary.Choice(f"Tests ({cfg.test.runner or 'none detected'})", "test", checked=cfg.test.enabled),
+        questionary.Choice("Dependency audit", "audit", checked=cfg.security.audit),
+        *repository_scans,
+        questionary.Choice(f"Build ({cfg.build.command or 'none detected'})", "build", checked=cfg.build.enabled),
+        questionary.Choice("Docker image (hadolint, build, Trivy scan)", "docker", checked=det.has_dockerfile),
+        *(
+            [questionary.Choice("CodeQL (needs GitHub Advanced Security on private repos)", "codeql", checked=False)]
+            if cfg.platform == "github" and cfg.language.ci.codeql_language
+            else []
+        ),
+        questionary.Choice("SonarCloud", "sonar", checked=False),
+    ]
+    selected = ask(questionary.checkbox("Jobs", choices=jobs))
+
+    data["lint"]["enabled"] = "lint" in selected
+    if data["lint"]["enabled"]:
+        known = cfg.lint_tools
+        options = [t for t in cfg.lint.tools if t not in known] + known
+        data["lint"]["tools"] = ask(questionary.checkbox(
+            "Lint tools",
+            choices=[questionary.Choice(t, t, checked=t in cfg.lint.tools) for t in options],
+        ))
+        data["lint"]["enabled"] = bool(data["lint"]["tools"])
+
+    data["test"]["enabled"] = "test" in selected and bool(cfg.test.runner)
+    if "test" in selected and not cfg.test.runner:
+        console.print("  [yellow]No test runner detected — set test.runner in quoll.toml to enable tests.[/yellow]")
+    if data["test"]["enabled"]:
+        if cfg.test.coverage:
+            threshold = ask(questionary.text(
+                "Minimum coverage (%)",
+                default=str(cfg.test.coverage_threshold),
+                validate=lambda v: v.isdigit() and 0 <= int(v) <= 100 or "Enter a number between 0 and 100",
+            ))
+            data["test"]["coverage_threshold"] = int(threshold)
+            data["test"]["codecov"] = ask(questionary.confirm("Upload coverage to Codecov?", default=False))
+        if len(det.matrix_versions) > 1:
+            versions = ", ".join(det.matrix_versions)
+            if ask(questionary.confirm(f"Test against several versions ({versions})?", default=False)):
+                data["versions"] = list(det.matrix_versions)
+        if det.services:
+            data["services"] = ask(questionary.checkbox(
+                "Start these services for the tests",
+                choices=[questionary.Choice(s, s, checked=True) for s in det.services],
+            ))
+        if det.env_vars:
+            platform = platforms.get(cfg.platform)
+            data["test"]["env_secrets"] = ask(questionary.checkbox(
+                f"Expose these .env.example variables to tests (as {platform.label} {platform.secrets_label})",
+                choices=[questionary.Choice(k, k, checked=True) for k in det.env_vars],
+            ))
+
+    data["security"]["audit"] = "audit" in selected
+    if not workspace:
+        data["security"]["secrets"] = "secrets" in selected
+        data["security"]["sast"] = "sast" in selected
+    data["security"]["codeql"] = "codeql" in selected
+    data["security"]["sonar"] = "sonar" in selected
+    if data["security"]["sonar"]:
+        owner = det.remote.path.split("/")[0] if det.remote else ""
+        data["security"]["sonar_organization"] = ask(questionary.text("SonarCloud organization", default=owner))
+        if det.remote:
+            data["security"]["sonar_project_key"] = det.remote.path.replace("/", "_")
+
+    data["build"]["enabled"] = "build" in selected and bool(cfg.build.command)
+    data["build"]["docker"] = "docker" in selected
+    return CIConfig.model_validate(data)
+
+
+def _remote_setup(repo: Repo, configs: List[CIConfig], det: Detection) -> None:
+    platform = platforms.get(repo.platform)
+    remote = det.remote
+    if remote is None or platforms.hosting(remote) != platform.name:
+        console.print(f"\n[dim]No {platform.label} remote (origin) found — skipping branch protection and "
+                      f"{platform.secrets_label}.[/dim]")
+        return
+    branch = repo.branches[0]
+
+    protect = ask(questionary.confirm(
+        f"Require the pipeline to pass before merging into {remote.path}@{branch}?", default=True
+    ))
+
+    secrets: Dict[str, str] = {}
+    wanted: List[str] = list(dict.fromkeys(s for cfg in configs for s in cfg.test.env_secrets))
+    if any(cfg.security.sonar for cfg in configs):
+        wanted.append("SONAR_TOKEN")
+    if any(cfg.test.codecov for cfg in configs):
+        wanted.append("CODECOV_TOKEN")
+    if wanted and ask(questionary.confirm(
+        f"Set the {platform.label} {platform.secrets_label} now ({', '.join(wanted)})?", default=True
+    )):
+        for key in wanted:
+            default = det.env_vars.get(key, "")
+            value = ask(questionary.password(f"{key}" + (" (Enter = .env.example value)" if default else "")))
+            if value or default:
+                secrets[key] = value or default
+
+    schedule = repo.schedule.enabled and platform.schedules_via_api and ask(questionary.confirm(
+        f"Schedule the security scans ({repo.schedule.cron}, UTC) on {branch}?", default=True
+    ))
+
+    if not protect and not secrets and not schedule:
+        return
+    if not ensure_token(platform.name, remote.host):
+        console.print(f"[yellow]No {platform.label} token — skipped.[/yellow]")
+        return
+
+    api = platform.api(remote.host)
+    if protect:
+        try:
+            for warning in api.protect_branch(remote.path, branch, required_checks=platform.required_checks):
+                console.print(f"[yellow]Warning:[/yellow] {warning}")
+            console.print(f"[green]✓[/green] {branch} is protected: merges require a green pipeline")
+        except PlatformError as e:
+            console.print(f"[yellow]Warning:[/yellow] branch protection failed: {e}")
+    if schedule:
+        _schedule(api, remote.path, branch, repo.schedule.cron)
+    if secrets:
+        try:
+            api.set_secrets(remote.path, secrets)
+            console.print(f"[green]✓[/green] {len(secrets)} {platform.secrets_label} set: {', '.join(secrets)}")
+        except Exception as e:  # httpx errors, missing PyNaCl...
+            console.print(f"[yellow]Warning:[/yellow] setting {platform.secrets_label} failed: {e}")
+
+
+def _schedule(api: PlatformAPI, repo: str, branch: str, cron: str) -> None:
+    try:
+        api.ensure_schedule(repo, branch, cron)
+        console.print(f"[green]✓[/green] Security scans scheduled on {branch} ({cron}, UTC)")
+    except PlatformError as e:
+        console.print(f"[yellow]Warning:[/yellow] scheduling the security scans failed: {e}")
+
+
+# --------------------------------------------------------------------------- output
+
+
+def _print_detection(det: Detection) -> None:
+    table = Table(show_header=False, box=None, padding=(0, 2))
+    table.add_column(style="dim")
+    table.add_column(style="bold")
+    version = f"{det.version} (from {det.version_file})" if det.version_file else det.version
+    table.add_row("Stack", det.stack or "-")
+    table.add_row("Framework", det.framework or "-")
+    table.add_row("Package manager", det.package_manager + (f" ({det.lockfile})" if det.lockfile else ""))
+    table.add_row("Version", version)
+    table.add_row("Lint", ", ".join(det.lint_tools) or "-")
+    table.add_row("Tests", det.test_runner + ("" if det.coverage or not det.test_runner else " (no coverage)") or "-")
+    table.add_row("Build", det.build_command.replace("\n", " && ") or "-")
+    table.add_row("Services", ", ".join(det.services) or "-")
+    table.add_row("Dockerfile", "yes" if det.has_dockerfile else "no")
+    table.add_row("Default branch", det.default_branch)
+    remote = f" ({det.remote.host}/{det.remote.path})" if det.remote else ""
+    table.add_row("Platform", platforms.get(det.platform).label + remote)
+    console.print(table)
+    for warning in det.warnings:
+        console.print(f"[yellow]![/yellow] {warning}")
+    console.print()
+
+
+# --------------------------------------------------------------------------- helpers
+
+
+def _plan(root: Path, target: Union[CIConfig, Repo]) -> list:
+    platform = target.platform
+    try:
+        return platforms.get(platform).ci_files(root, target)
+    except ValueError as e:
+        console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)
+
+
+def _repo(project_dir: Path) -> Tuple[Path, Repo]:
+    """The repository `project_dir` belongs to (its workspace root, if any) and its pipeline settings."""
+    root = find_root(project_dir)
+    try:
+        return root, load_repo(root)
+    except ConfigError as e:
+        console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)
+
+
+def _configured(project_dir: Path) -> bool:
+    try:
+        data = quoll_toml.read(project_dir)
+    except ConfigError:
+        return True
+    return "ci" in data or "workspace" in data