quoll-cli 0.2.0__py3-none-any.whl

quoll/services.py

@@ -0,0 +1,117 @@
+"""Backing services (databases, caches) an app needs: containers for CI tests and for compose.yaml.
+
+Languages detect which ones a project uses (see `detect_services`); this module knows how to run them and
+which environment variables point an app at them. Credentials are throwaway: these containers only serve
+tests and local development.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Dict, Iterable, List, Mapping, Union
+
+DATABASE = "app"
+
+
+@dataclass(frozen=True)
+class Service:
+    name: str
+    image: str
+    port: int
+    env: Dict[str, str]
+    # Shell command that succeeds once the service accepts connections.
+    health: str
+    # Where the container keeps its data (a named volume in compose.yaml).
+    data_dir: str
+    # Environment variables pointing an app at the service on `host`, using `database` (a name of its own).
+    app_env: Callable[[str, str], Dict[str, str]]
+
+
+def _postgres(host: str, database: str) -> Dict[str, str]:
+    url = f"postgresql://{DATABASE}:{DATABASE}@{host}:5432/{database}"
+    # The PG* variables are read by libpq-based clients (psycopg, pg, rails...) without any configuration.
+    return {
+        "DATABASE_URL": url,
+        "PGHOST": host,
+        "PGPORT": "5432",
+        "PGUSER": DATABASE,
+        "PGPASSWORD": DATABASE,
+        "PGDATABASE": database,
+    }
+
+
+def _mysql(host: str, database: str) -> Dict[str, str]:
+    # root, so test runners can create their test databases (Django, Rails...).
+    return {"DATABASE_URL": f"mysql://root:root@{host}:3306/{database}"}
+
+
+SERVICES: Dict[str, Service] = {
+    s.name: s
+    for s in (
+        Service(
+            name="postgres",
+            image="postgres:17-alpine",
+            port=5432,
+            env={"POSTGRES_USER": DATABASE, "POSTGRES_PASSWORD": DATABASE, "POSTGRES_DB": DATABASE},
+            health=f"pg_isready -U {DATABASE} -d {DATABASE}",
+            data_dir="/var/lib/postgresql/data",
+            app_env=_postgres,
+        ),
+        Service(
+            name="mysql",
+            image="mysql:8.4",
+            port=3306,
+            env={"MYSQL_ROOT_PASSWORD": "root", "MYSQL_DATABASE": DATABASE},
+            health="mysqladmin ping -h 127.0.0.1 -uroot -proot",
+            data_dir="/var/lib/mysql",
+            app_env=_mysql,
+        ),
+        Service(
+            name="redis",
+            image="redis:7-alpine",
+            port=6379,
+            env={},
+            health="redis-cli ping",
+            data_dir="/data",
+            app_env=lambda host, database: {"REDIS_URL": f"redis://{host}:6379"},
+        ),
+        Service(
+            name="mongodb",
+            image="mongo:8",
+            port=27017,
+            env={},
+            health="mongosh --quiet --eval 'db.adminCommand({ping: 1})'",
+            data_dir="/data/db",
+            app_env=lambda host, database: {"MONGODB_URI": f"mongodb://{host}:27017/{database}"},
+        ),
+    )
+}
+
+# Signature of the language/framework hooks adding their own variables (Laravel's DB_*, Spring's datasource...):
+# (service, host, database) -> variables.
+ServiceEnv = Callable[[str, str, str], Dict[str, str]]
+
+
+def no_service_env(service: str, host: str, database: str) -> Dict[str, str]:
+    return {}
+
+
+def detect_services(dependencies: Iterable[str], packages: Mapping[str, Iterable[str]]) -> List[str]:
+    """Services whose client packages appear in `dependencies`, in catalog order."""
+    names = {d.lower() for d in dependencies}
+    return [service for service in SERVICES if any(p.lower() in names for p in packages.get(service, ()))]
+
+
+def environment(
+    services: Iterable[str], host: Union[str, Callable[[str], str]], *hooks: ServiceEnv, database: str = DATABASE
+) -> Dict[str, str]:
+    """Variables pointing an app at `services`, reachable on `host` (or `host(service)`), using its `database`.
+
+    Later hooks override earlier values.
+    """
+    env: Dict[str, str] = {}
+    for name in services:
+        address = host(name) if callable(host) else host
+        env.update(SERVICES[name].app_env(address, database))
+        for hook in hooks:
+            env.update(hook(name, address, database))
+    return env

quoll/starters.py

@@ -0,0 +1,117 @@
+"""Templates and generators (`ng new`, `npm install`...) behind the starters of `quoll new`.
+
+A generator runs with the local tool when it is installed and no runtime version is asked for; otherwise in a
+container of that runtime's image, so that the project and its lockfile match the version it will run on.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from typing import List, Optional, Sequence
+
+from quoll.languages.base import StarterOptions
+
+# Runtimes' container images, by runtime and version.
+IMAGES = {"node": "node:{version}"}
+
+
+class StarterError(Exception):
+    pass
+
+
+def run(command: List[str], cwd: Path, runtime: str = "", version: str = "") -> None:
+    """Run `command` in `cwd`: in the `runtime:version` image when a version is asked for or the tool is missing."""
+    image = IMAGES[runtime].format(version=version or "latest") if runtime else ""
+    if shutil.which(command[0]) and not version:
+        full = command
+    elif image and shutil.which("docker"):
+        # The project's files stay the user's; caches go to the container's /tmp.
+        user = ["--user", f"{os.getuid()}:{os.getgid()}"] if hasattr(os, "getuid") else []
+        full = ["docker", "run", "--rm", "-v", f"{cwd}:/work", "-w", "/work", *user, "-e", "HOME=/tmp", image, *command]
+    else:
+        needed = f"Docker (to run {image})" if version else f"`{command[0]}` (or Docker)"
+        raise StarterError(f"{needed} is needed to create this project")
+    result = subprocess.run(full, cwd=cwd, capture_output=True, text=True)
+    if result.returncode != 0:
+        output = (result.stdout + result.stderr).strip().splitlines()
+        raise StarterError(f"`{' '.join(command)}` failed:\n  " + "\n  ".join(output[-15:]))
+
+
+def in_container(command: str, version: str) -> bool:
+    """Whether `run` runs `command` in a container (a runtime version is asked for, or the tool is missing)."""
+    return bool(version) or not shutil.which(command)
+
+
+def install_locally(directory: Path, version: str, command: Sequence[str], folder: str) -> List[str]:
+    """Dependencies installed in a (Linux) container hold its native binaries: reinstall them for this machine
+    from the lock file, or leave `folder` out. Returns a note for the user, if any."""
+    if not in_container(command[0], version):
+        return []
+    shutil.rmtree(directory / folder, ignore_errors=True)
+    if shutil.which(command[0]):
+        result = subprocess.run(list(command), cwd=directory, capture_output=True, text=True)
+        if result.returncode == 0:
+            return []
+    return [f"{directory.name}: run `{' '.join(command)}` there before working locally."]
+
+
+def write(directory: Path, files: dict) -> None:
+    for rel, content in files.items():
+        path = directory / rel
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(content)
+
+
+def from_template(
+    template: Path,
+    commands: Sequence[Sequence[str]] = (),
+    *,
+    runtime: str = "",
+    runtime_default: str = "",
+    version_default: str = "latest",
+    local_install: Sequence[str] = (),
+    local_folder: str = "",
+    renamed: Optional[dict] = None,
+):
+    """A starter's `create` copying `template`, then running `commands` in the project.
+
+    In files and commands, {{name}}, {{version}} (the framework's) and {{runtime}} (the runtime's) are replaced.
+    When the commands ran in a container, `local_install` reinstalls `local_folder` for this machine.
+    Templates live next to their plugin, in templates/<starter>/. Dot files are stored without their dot (see
+    `renamed`), so they don't apply to quoll's own repository.
+    """
+    renamed = {
+        "gitignore": ".gitignore",
+        "env.example": ".env.example",
+        "nvmrc": ".nvmrc",
+        "prettierignore": ".prettierignore",
+        **(renamed or {}),
+    }
+
+    def create(directory: Path, options: StarterOptions) -> List[str]:
+        values = {
+            "{{name}}": options.name,
+            "{{version}}": options.version or version_default,
+            "{{runtime}}": options.runtime or runtime_default,
+        }
+
+        def fill(text: str) -> str:
+            for key, value in values.items():
+                text = text.replace(key, value)
+            return text
+
+        files = {}
+        for path in sorted(template.rglob("*")):
+            if path.is_file():
+                rel = path.relative_to(template).as_posix()
+                files[renamed.get(rel, rel)] = fill(path.read_text())
+        write(directory, files)
+        for command in commands:
+            run([fill(part) for part in command], directory, runtime, options.runtime)
+        if local_install:
+            return install_locally(directory, options.runtime, local_install, local_folder)
+        return []
+
+    return create

quoll/workspace.py

@@ -0,0 +1,232 @@
+"""Repositories holding several projects (a front-end and a back-end, services...), each with its own quoll.toml.
+
+The root quoll.toml lists them and holds what the repository shares; every member keeps the format of a
+single project, so a member taken out of the repository stays a valid quoll project:
+
+    # quoll.toml
+    [project]
+    platform = "github"
+
+    [workspace]
+    members = ["backend", "frontend"]
+
+    [ci]                       # the repository's: when pipelines run, repository-wide scans
+    branches = ["main"]
+
+    # backend/quoll.toml, frontend/quoll.toml: [project], [ci], [docker], [cd] as usual
+
+A repository with a single project is the one-member case: everything below works for both.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional
+
+from pydantic import Field, model_validator
+
+from quoll.cd.config import CDConfig
+from quoll.cd.config import load_config as load_cd_config
+from quoll.ci.config import CIConfig, ScheduleConfig, load_config
+from quoll.config.quoll_toml import CONFIG_FILE, ConfigError, PlatformField, Section, load_section, read, save_section
+from quoll.k8s.config import K8sConfig
+from quoll.k8s.config import load_config as load_k8s_config
+
+WORKSPACE_SECTION = "workspace"
+
+
+class WorkspaceConfig(Section):
+    # Directories of the projects, relative to the repository root ("." for a project at the root).
+    members: List[str]
+
+    @model_validator(mode="after")
+    def _check(self) -> "WorkspaceConfig":
+        if not self.members:
+            raise ValueError("workspace.members must list at least one project directory")
+        names = [member_name(m) for m in self.members]
+        duplicates = sorted({n for n in names if names.count(n) > 1})
+        if duplicates:
+            raise ValueError(f"two members would be named '{duplicates[0]}': rename one of their directories")
+        return self
+
+
+class RepoSecurity(Section):
+    # Scans of the whole repository, run once whatever the members.
+    secrets: bool = True
+    sast: bool = True
+
+
+class RepoCIConfig(Section, PlatformField):
+    """[ci] of a workspace root: what the members share."""
+
+    shared_fields = ("platform",)
+
+    branches: List[str] = Field(default_factory=lambda: ["main"])
+    dependency_updates: bool = True
+    schedule: ScheduleConfig = Field(default_factory=ScheduleConfig)
+    security: RepoSecurity = Field(default_factory=RepoSecurity)
+
+
+@dataclass
+class Member:
+    # Job prefix: the last part of its directory ("" for a single project).
+    name: str
+    cfg: CIConfig
+
+    @property
+    def directory(self) -> str:
+        return self.cfg.directory
+
+    def job(self, job: str) -> str:
+        return f"{self.name}-{job}" if self.name else job
+
+
+@dataclass
+class Delivery:
+    """The delivery of one project: [cd] of a single project ("" name), or of a workspace member."""
+
+    name: str
+    directory: str
+    cfg: CDConfig
+    # Its Helm chart, deployed to each environment once the image is published.
+    k8s: Optional[K8sConfig] = None
+
+
+@dataclass
+class Repo:
+    """Everything one pipeline covers: the repository's settings and its projects."""
+
+    platform: str
+    branches: List[str]
+    dependency_updates: bool
+    schedule: ScheduleConfig
+    secrets: bool
+    sast: bool
+    members: List[Member]
+
+    @property
+    def workspace(self) -> bool:
+        return any(m.name for m in self.members)
+
+    @classmethod
+    def single(cls, cfg: CIConfig) -> "Repo":
+        return cls(
+            platform=cfg.platform,
+            branches=list(cfg.branches),
+            dependency_updates=cfg.dependency_updates,
+            schedule=cfg.schedule,
+            secrets=cfg.security.secrets,
+            sast=cfg.security.sast,
+            members=[Member("", cfg)],
+        )
+
+
+# Directories that hold no project of their own, even with a requirements.txt or a package.json (docs, examples).
+_NOT_PROJECTS = {
+    "docs", "doc", "examples", "example", "samples", "test", "tests", "fixtures", "scripts", "node_modules",
+    "vendor", "dist", "build", "target", "venv", ".venv",
+}
+# Where monorepos usually keep their projects, one level deeper.
+_CONTAINERS = ("apps", "packages", "services")
+
+
+def discover(root: Path) -> List[str]:
+    """Project directories of the repository: the root, its first-level directories and apps/*, packages/*,
+    services/* — each where quoll recognizes a language."""
+    from quoll import languages
+    from quoll.project import Project
+
+    candidates = [root]
+    for child in sorted(root.iterdir()):
+        if not child.is_dir() or child.name.startswith(".") or child.name in _NOT_PROJECTS:
+            continue
+        if child.name in _CONTAINERS:
+            candidates += sorted(c for c in child.iterdir() if c.is_dir() and not c.name.startswith("."))
+        else:
+            candidates.append(child)
+    found = []
+    for directory in candidates:
+        if languages.matching(Project(directory)):
+            found.append(directory.relative_to(root).as_posix() or ".")
+    return found
+
+
+def member_name(directory: str) -> str:
+    """backend -> backend, apps/web -> web, . -> root."""
+    parts = [p for p in directory.strip("/").split("/") if p and p != "."]
+    return parts[-1] if parts else "root"
+
+
+def find_root(start: Path) -> Path:
+    """The workspace root `start` belongs to (itself or a parent listing it), else `start`."""
+    start = start.resolve()
+    for candidate in [start, *start.parents]:
+        try:
+            data = read(candidate)
+        except ConfigError:
+            continue
+        workspace = data.get(WORKSPACE_SECTION)
+        if workspace is None:
+            continue
+        members = [(candidate / m).resolve() for m in workspace.get("members", [])]
+        if candidate == start or start in members:
+            return candidate
+    return start
+
+
+def load_workspace(root: Path) -> Optional[WorkspaceConfig]:
+    data = read(root)
+    if WORKSPACE_SECTION not in data:
+        return None
+    return load_section(root, WORKSPACE_SECTION, WorkspaceConfig)
+
+
+def load_repo(root: Path) -> Repo:
+    """The repository's pipeline settings: a workspace's, or a single project's (its [ci])."""
+    workspace = load_workspace(root)
+    if workspace is None:
+        return Repo.single(load_config(root))
+    shared = load_section(root, "ci", RepoCIConfig)
+    members = []
+    for directory in workspace.members:
+        cfg = load_config(root / directory)
+        if cfg.platform != shared.platform:
+            raise ConfigError(
+                f"{directory}/{CONFIG_FILE} targets {cfg.platform}, the workspace {shared.platform}: "
+                "set the same [project] platform"
+            )
+        members.append(Member(member_name(directory), cfg.in_directory(directory)))
+    return Repo(
+        platform=shared.platform,
+        branches=list(shared.branches),
+        dependency_updates=shared.dependency_updates,
+        schedule=shared.schedule,
+        secrets=shared.security.secrets,
+        sast=shared.security.sast,
+        members=members,
+    )
+
+
+def save_workspace(root: Path, members: List[str], shared: RepoCIConfig) -> None:
+    save_section(root, WORKSPACE_SECTION, WorkspaceConfig(members=members))
+    save_section(root, "ci", shared)
+
+
+def load_deliveries(root: Path) -> List[Delivery]:
+    """The [cd] of a single project, or of each workspace member that has one."""
+    workspace = load_workspace(root)
+    if workspace is None:
+        return [Delivery("", ".", load_cd_config(root), _chart(root))]
+    deliveries = []
+    for directory in workspace.members:
+        if "cd" in read(root / directory):
+            deliveries.append(
+                Delivery(member_name(directory), directory, load_cd_config(root / directory), _chart(root / directory))
+            )
+    if not deliveries:
+        raise ConfigError("no workspace member has a [cd] section. Run `quoll cd init` in one of them first.")
+    return deliveries
+
+
+def _chart(directory: Path) -> Optional[K8sConfig]:
+    return load_k8s_config(directory) if "k8s" in read(directory) else None

quoll/yaml_dump.py

@@ -0,0 +1,137 @@
+"""Minimal, deterministic YAML emitter (GitHub workflows, GitLab pipelines, compose files).
+
+quoll builds documents as plain dicts and emits them itself so the output is stable (byte-for-byte, which the
+`check` commands rely on), readable and prettier-compatible.
+"""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Dict, Iterable, List
+
+
+class Commented(str):
+    """A scalar followed by a comment, e.g. an action pinned to a commit, with its version."""
+
+    comment: str
+
+    def __new__(cls, value: str, comment: str) -> "Commented":
+        obj = super().__new__(cls, value)
+        obj.comment = comment
+        return obj
+
+
+_PLAIN_RE = re.compile(r"^[A-Za-z_][\w ./@()=,+-]*$")
+_YAML_KEYWORDS = {"true", "false", "yes", "no", "on", "off", "null", "y", "n", "~"}
+_RUN_UNSAFE_START = set("-?:,[]{}#&*!|>'\"%@`")
+
+
+def dump(doc: Dict[str, Any], spaced: Iterable[str] = (), scripts: Iterable[str] = ("run",)) -> str:
+    """YAML for `doc`; children of the `spaced` mappings are separated by a blank line, for readability.
+
+    Values of the `scripts` keys are shell commands (a string or a list of them): emitted unquoted when YAML allows,
+    as literal blocks otherwise.
+    """
+    lines: List[str] = []
+    top_keys = list(doc)
+    for i, key in enumerate(top_keys):
+        if i:
+            lines.append("")
+        lines.extend(_emit_entry(key, doc[key], 0, frozenset(spaced), frozenset(scripts)))
+    return "\n".join(lines) + "\n"
+
+
+def _emit_entry(key: str, value: Any, indent: int, spaced: frozenset, scripts: frozenset) -> List[str]:
+    pad = " " * indent
+    if isinstance(value, Commented):
+        return [f"{pad}{key}: {_scalar(str(value))} # {value.comment}"]
+    if key in scripts and isinstance(value, list):
+        out = [f"{pad}{key}:"]
+        for command in value:
+            out.extend(_emit_command(f"{pad}  - ", command, indent + 4))
+        return out
+    if key in scripts and isinstance(value, str):
+        return _emit_command(f"{pad}{key}: ", value, indent + 2)
+    if isinstance(value, dict):
+        if not value:
+            return [f"{pad}{key}: {{}}"]
+        out = [f"{pad}{key}:"]
+        for i, (k, v) in enumerate(value.items()):
+            if i and key in spaced:
+                out.append("")
+            out.extend(_emit_entry(k, v, indent + 2, spaced, scripts))
+        return out
+    if isinstance(value, list):
+        if all(not isinstance(v, (dict, list)) for v in value):
+            return [f"{pad}{key}: [{', '.join(_scalar(v) for v in value)}]"]
+        out = [f"{pad}{key}:"]
+        for i, item in enumerate(value):
+            if i and key in spaced:
+                out.append("")
+            out.extend(_emit_list_item(item, indent + 2, spaced, scripts))
+        return out
+    if value is None:
+        return [f"{pad}{key}:"]
+    if isinstance(value, str) and "\n" in value:
+        return _block(f"{pad}{key}: ", value, indent + 2)
+    return [f"{pad}{key}: {_scalar(value)}"]
+
+
+def _emit_command(prefix: str, command: str, indent: int) -> List[str]:
+    if "\n" in command or not _plain_run(command):
+        return _block(prefix, command, indent)
+    return [f"{prefix}{command}"]
+
+
+def _block(prefix: str, text: str, indent: int) -> List[str]:
+    pad = " " * indent
+    return [f"{prefix}|", *(f"{pad}{line}" if line else "" for line in text.split("\n"))]
+
+
+def _emit_list_item(item: Any, indent: int, spaced: frozenset, scripts: frozenset) -> List[str]:
+    pad = " " * indent
+    if not isinstance(item, dict):
+        return [f"{pad}- {_scalar(item)}"]
+    out: List[str] = []
+    for i, (k, v) in enumerate(item.items()):
+        entry = _emit_entry(k, v, indent + 2, spaced, scripts)
+        if i == 0:
+            entry[0] = f"{pad}- {entry[0].lstrip()}"
+        out.extend(entry)
+    return out
+
+
+def _scalar(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if value is None:
+        return "null"
+    text = str(value)
+    if text.startswith("${{") and ": " not in text and " #" not in text:
+        return text
+    if _plain_scalar(text):
+        return text
+    return json.dumps(text)
+
+
+def _plain_scalar(text: str) -> bool:
+    """Strings that read the same unquoted: start with a letter, no YAML indicators inside."""
+    return (
+        bool(_PLAIN_RE.match(text))
+        and text.lower() not in _YAML_KEYWORDS
+        and ": " not in text
+        and " #" not in text
+        and not text.endswith((":", " "))
+    )
+
+
+def _plain_run(cmd: str) -> bool:
+    return (
+        bool(cmd)
+        and cmd[0] not in _RUN_UNSAFE_START
+        and ": " not in cmd
+        and " #" not in cmd
+        and not cmd.endswith(":")
+    )