quoll-cli 0.2.0__py3-none-any.whl

quoll/k8s/config.py

@@ -0,0 +1,97 @@
+"""The [k8s] section of quoll.toml — the single source of truth for the generated Helm chart and its deployment."""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Dict, List
+
+from pydantic import Field, model_validator
+
+from quoll.config.quoll_toml import CONFIG_FILE, ConfigError, Section, load_section, save_section
+
+__all__ = ["CONFIG_FILE", "ConfigError", "Environment", "K8sConfig", "load_config", "save_config"]
+
+# What deploys an environment: every push to the [cd] branches, or the v* tags (after a manual approval).
+TRIGGERS = ("branch", "tag")
+_NAME = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
+
+
+class Environment(Section):
+    # Created when missing. Every project of a workspace deploys to the same one, so that they reach each other.
+    namespace: str
+    deploy_on: str = "branch"
+    # Public host of the Ingress, e.g. "api.example.com". Empty: reachable inside the cluster only.
+    host: str = ""
+    replicas: int = Field(default=1, ge=1)
+
+    @model_validator(mode="after")
+    def _check(self) -> "Environment":
+        if self.deploy_on not in TRIGGERS:
+            raise ValueError(f"deploy_on must be one of: {', '.join(TRIGGERS)}")
+        if not _NAME.match(self.namespace):
+            raise ValueError(f"namespace '{self.namespace}' isn't a valid Kubernetes name (lowercase, digits, -)")
+        return self
+
+
+class K8sConfig(Section):
+    # Helm release, Service and Deployment name: the other projects reach this one at http://<name>:<port>.
+    name: str
+    # Directory of the Helm chart, in the project.
+    chart: str = "chart"
+    # HTTP path answering 200 once the app is up, e.g. "/health". Empty: TCP checks on the port.
+    health_path: str = ""
+    cpu: str = "100m"
+    memory: str = "256Mi"
+    memory_limit: str = "512Mi"
+    # HorizontalPodAutoscaler between an environment's replicas and this (0: off).
+    max_replicas: int = Field(default=0, ge=0)
+    # Ingress: the cluster's default class when empty; TLS certificate from cert-manager when cluster_issuer is set.
+    ingress_class: str = ""
+    tls: bool = True
+    cluster_issuer: str = ""
+    # Plain environment variables (secrets go in the Secret `secret`, created in the cluster).
+    env: Dict[str, str] = Field(default_factory=dict)
+    # Secret holding the app's secrets as environment variables. Empty: "<name>-env".
+    secret: str = ""
+    # Secret to pull a private image with (a docker-registry Secret in the namespace).
+    pull_secret: str = ""
+    environments: Dict[str, Environment] = Field(default_factory=dict)
+
+    omit_when_empty = ("env",)
+
+    @model_validator(mode="after")
+    def _check(self) -> "K8sConfig":
+        if not _NAME.match(self.name):
+            raise ValueError(f"name '{self.name}' isn't a valid Kubernetes name (lowercase, digits, -)")
+        if self.health_path and not self.health_path.startswith("/"):
+            raise ValueError("health_path should start with /, e.g. /health")
+        return self
+
+    @property
+    def secret_name(self) -> str:
+        return self.secret or f"{self.name}-env"
+
+    def deployed_on(self, trigger: str) -> List[str]:
+        return [name for name, env in self.environments.items() if env.deploy_on == trigger]
+
+
+def default_environments(project: str) -> Dict[str, Environment]:
+    """Staging on every push to the delivered branches, production on v* tags."""
+    return {
+        "staging": Environment(namespace=f"{project}-staging"),
+        "production": Environment(namespace=project, deploy_on="tag", replicas=2),
+    }
+
+
+def kubernetes_name(value: str) -> str:
+    """`value` as a Kubernetes name: lowercase letters, digits and dashes."""
+    name = re.sub(r"[^a-z0-9-]+", "-", value.lower()).strip("-")
+    return name[:53] or "app"
+
+
+def load_config(project_dir: Path) -> K8sConfig:
+    return load_section(project_dir, "k8s", K8sConfig)
+
+
+def save_config(project_dir: Path, config: K8sConfig) -> Path:
+    return save_section(project_dir, "k8s", config)

quoll/k8s/deploy.py

@@ -0,0 +1,22 @@
+"""Deploying a published image with its Helm chart: the commands both platforms' delivery jobs run."""
+from __future__ import annotations
+
+from quoll.k8s.config import Environment, K8sConfig
+
+# What the deploy jobs read: the image the delivery pushed, by digest (what was scanned, signed and attested).
+IMAGE, DIGEST = "IMAGE", "DIGEST"
+
+
+def helm_upgrade(k8s: K8sConfig, environment: str, settings: Environment) -> str:
+    """Install or upgrade the release, wait until it is ready; roll it back otherwise."""
+    return (
+        f"helm upgrade --install {k8s.name} {k8s.chart} -f {k8s.chart}/values-{environment}.yaml \\\n"
+        f"  --namespace {settings.namespace} --create-namespace \\\n"
+        f'  --set image.repository="${IMAGE}" --set image.digest="${DIGEST}" \\\n'
+        "  --rollback-on-failure --timeout 10m"
+    )
+
+
+def url(settings: Environment, tls: bool) -> str:
+    """The environment's address, shown by the platform next to the deployment."""
+    return f"{'https' if tls else 'http'}://{settings.host}" if settings.host else ""

quoll/k8s/detect.py

@@ -0,0 +1,32 @@
+"""What quoll can tell about a project's deployment from its code."""
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+
+_SOURCES = (".ts", ".js", ".mjs", ".py", ".go", ".rb", ".php", ".java", ".kt", ".cs", ".rs")
+_SKIPPED = {"node_modules", "vendor", ".venv", "venv", "dist", "build", "target", ".git", "bin", "obj", "test", "tests"}
+# Routes health checks usually answer on, most specific first.
+_HEALTH = re.compile(r"""["'](/(?:healthz|health|livez|readyz|ping|up))["']""")
+_MAX_FILES = 500
+
+
+def health_path(project_dir: Path) -> str:
+    """The health route the app's code declares, e.g. "/health"; empty when none is found."""
+    seen = 0
+    for directory, subdirectories, files in os.walk(project_dir):
+        subdirectories[:] = sorted(d for d in subdirectories if d not in _SKIPPED and not d.startswith("."))
+        for name in sorted(files):
+            if not name.endswith(_SOURCES) or ".test." in name or ".spec." in name:
+                continue
+            seen += 1
+            if seen > _MAX_FILES:
+                return ""
+            try:
+                match = _HEALTH.search((Path(directory) / name).read_text(errors="ignore"))
+            except OSError:
+                continue
+            if match:
+                return match.group(1)
+    return ""

quoll/k8s/render.py

@@ -0,0 +1,94 @@
+"""Turn a K8sConfig, with the project's [docker] (port, image) and [cd] (image name), into a Helm chart.
+
+The templates are the same for every project (quoll/k8s/chart/templates); what differs is in the values files:
+values.yaml for everything, values-<environment>.yaml for what an environment changes (replicas, host)."""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from quoll.docker.config import DockerConfig
+from quoll.files import PlannedFile, header, planned
+from quoll.k8s.config import Environment, K8sConfig
+from quoll.yaml_dump import dump
+
+TEMPLATES = Path(__file__).parent / "chart" / "templates"
+_HELMIGNORE = "# Patterns kept out of the chart package.\n.DS_Store\n*.swp\n*.bak\n*.tmp\n"
+
+
+def render_chart_yaml(k8s: K8sConfig) -> str:
+    doc = {
+        "apiVersion": "v2",
+        "name": k8s.name,
+        "description": f"{k8s.name}, generated by quoll",
+        "type": "application",
+        "version": "0.1.0",
+        "appVersion": "latest",
+    }
+    return header("k8s") + dump(doc)
+
+
+def render_values(
+    k8s: K8sConfig, docker: DockerConfig, image: str = "", peers: Optional[Dict[str, str]] = None
+) -> str:
+    """values.yaml: `image` from [cd] (the deployment sets its digest), `peers` the URLs of the other projects."""
+    writable = docker.image.writable_paths(docker)
+    ingress: Dict[str, Any] = {"enabled": False, "className": k8s.ingress_class, "host": "", "tls": k8s.tls}
+    ingress["annotations"] = {"cert-manager.io/cluster-issuer": k8s.cluster_issuer} if k8s.cluster_issuer else {}
+    doc: Dict[str, Any] = {
+        # The deployment sets the digest of the image it published: what was scanned and signed runs.
+        "image": {"repository": image, "tag": "latest", "digest": "", "pullPolicy": "IfNotPresent"},
+        "imagePullSecrets": [k8s.pull_secret] if k8s.pull_secret else [],
+        "replicaCount": 1,
+        "port": docker.port,
+        "healthPath": k8s.health_path,
+        "env": {**(peers or {}), **k8s.env},
+        "envFromSecret": k8s.secret_name,
+        "resources": {
+            "requests": {"cpu": k8s.cpu, "memory": k8s.memory},
+            "limits": {"memory": k8s.memory_limit},
+        },
+        # The app only writes to these directories (emptyDir volumes); null: writes in its own tree.
+        "readOnlyRootFilesystem": writable is not None,
+        "writablePaths": writable or [],
+        "ingress": ingress,
+        "autoscaling": {
+            "enabled": k8s.max_replicas > 0,
+            "minReplicas": 1,
+            "maxReplicas": max(k8s.max_replicas, 1),
+            "targetCPUUtilizationPercentage": 80,
+        },
+    }
+    return header("k8s") + dump(doc)
+
+
+def render_environment(k8s: K8sConfig, environment: Environment) -> str:
+    """values-<environment>.yaml: what the environment changes."""
+    doc: Dict[str, Any] = {"replicaCount": environment.replicas}
+    if k8s.max_replicas:
+        doc["autoscaling"] = {"minReplicas": environment.replicas}
+    if environment.host:
+        doc["ingress"] = {"enabled": True, "host": environment.host}
+    return header("k8s") + dump(doc)
+
+
+def plan(
+    project_dir: Path,
+    k8s: K8sConfig,
+    docker: DockerConfig,
+    image: str = "",
+    peers: Optional[Dict[str, str]] = None,
+) -> List[PlannedFile]:
+    chart = Path(k8s.chart)
+    files = [
+        planned(project_dir, chart / "Chart.yaml", render_chart_yaml(k8s)),
+        planned(project_dir, chart / "values.yaml", render_values(k8s, docker, image, peers)),
+        *(
+            planned(project_dir, chart / f"values-{name}.yaml", render_environment(k8s, environment))
+            for name, environment in k8s.environments.items()
+        ),
+        planned(project_dir, chart / ".helmignore", header("k8s") + _HELMIGNORE),
+    ]
+    for template in sorted(TEMPLATES.iterdir()):
+        files.append(planned(project_dir, chart / "templates" / template.name, template.read_text()))
+    return files

quoll/languages/__init__.py

@@ -0,0 +1,121 @@
+"""Language and framework plugins registry.
+
+Built-in languages live in sub-packages of `quoll.languages`, each exposing a `language` object whose
+`frameworks` live in its `frameworks/` sub-package. Third-party packages add languages or frameworks
+(for any language) through entry points:
+
+    [project.entry-points."quoll.languages"]
+    elixir = "quoll_elixir:language"
+
+    [project.entry-points."quoll.frameworks"]
+    remix = "quoll_remix:framework"
+"""
+
+from __future__ import annotations
+
+import importlib
+from importlib.metadata import entry_points
+from typing import Dict, List, Optional
+
+from quoll.languages.base import DockerImage, Framework, Language, Starter
+from quoll.project import Project
+
+LANGUAGES_GROUP = "quoll.languages"
+FRAMEWORKS_GROUP = "quoll.frameworks"
+
+# Also the detection priority. Languages whose projects often carry a package.json for their front-end
+# assets (Rails, Symfony, Laravel, Django...) come before node. C/C++ comes last: a CMakeLists.txt also builds the
+# native parts of Python or Node packages.
+_BUILTIN = ("go", "rust", "java", "dotnet", "php", "ruby", "python", "node", "cpp")
+
+_languages: Dict[str, Language] = {}
+_frameworks: Dict[str, List[Framework]] = {}
+_loaded = False
+
+
+def register(language: Language) -> None:
+    if not isinstance(language, Language):
+        raise TypeError(f"expected a quoll.languages.base.Language, got {type(language).__name__}")
+    if language.name in _languages:
+        raise ValueError(f"language '{language.name}' is already registered")
+    _languages[language.name] = language
+    _frameworks[language.name] = []
+    for framework in language.frameworks:
+        register_framework(framework)
+
+
+def register_framework(framework: Framework) -> None:
+    if not isinstance(framework, Framework):
+        raise TypeError(f"expected a quoll.languages.base.Framework, got {type(framework).__name__}")
+    if framework.language not in _languages:
+        raise ValueError(f"framework '{framework.name}' targets unknown language '{framework.language}'")
+    if any(f.name == framework.name for f in _frameworks[framework.language]):
+        raise ValueError(f"framework '{framework.name}' is already registered for {framework.language}")
+    _frameworks[framework.language].append(framework)
+
+
+def available() -> List[Language]:
+    """Registered languages, in detection priority order."""
+    _load()
+    return list(_languages.values())
+
+
+def names() -> List[str]:
+    return [lang.name for lang in available()]
+
+
+def get(name: str) -> Language:
+    _load()
+    try:
+        return _languages[name]
+    except KeyError:
+        raise KeyError(f"unknown language '{name}' (available: {', '.join(names())})") from None
+
+
+def starters(role: str = "") -> List[Starter]:
+    """Every starter of the registered languages and frameworks (for `role` only, when given)."""
+    found = [s for lang in available() for owner in (lang, *frameworks(lang.name)) for s in owner.starters]
+    return [s for s in found if not role or s.role == role]
+
+
+def frameworks(language: str) -> List[Framework]:
+    """Frameworks of `language`, in detection priority order."""
+    _load()
+    return list(_frameworks.get(language, []))
+
+
+def get_framework(language: str, name: str) -> Optional[Framework]:
+    return next((f for f in frameworks(language) if f.name == name), None)
+
+
+def docker_image(language: str, framework: str = "") -> Optional[DockerImage]:
+    """The framework's image when it has one, else the language's."""
+    fw = get_framework(language, framework) if framework else None
+    return (fw.docker if fw else None) or get(language).docker
+
+
+def matching(project: Project) -> List[Language]:
+    """Languages whose marker files are present, highest priority first."""
+    return [lang for lang in available() if lang.matches(project)]
+
+
+def _load() -> None:
+    global _loaded
+    if _loaded:
+        return
+    _loaded = True
+    for name in _BUILTIN:
+        register(importlib.import_module(f"{__name__}.{name}").language)
+    for group, add in ((LANGUAGES_GROUP, register), (FRAMEWORKS_GROUP, register_framework)):
+        for ep in _entry_points(group):
+            try:
+                add(ep.load())
+            except Exception as e:
+                raise RuntimeError(f"quoll plugin '{ep.name}' ({ep.value}) failed to load: {e}") from e
+
+
+def _entry_points(group: str) -> list:
+    try:
+        return list(entry_points(group=group))
+    except TypeError:  # Python 3.9: entry_points() takes no arguments
+        return list(entry_points().get(group, []))