cloudfall 0.2.0__py3-none-any.whl

cloudfall_engine/ansible_inventory.py

@@ -0,0 +1,253 @@
+"""Render deterministic static Ansible inventory from typed Cloudfall inventory."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from cloudfall.inventory import firewall_rules_for_server
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from cloudfall.domain import ResourceId
+    from cloudfall.inventory import (
+        AlertRuleInventory,
+        ApplicationInventory,
+        ComponentInventory,
+        LoggingStackInventory,
+        PlatformInventory,
+        ServerInventory,
+        ServerTypeInventory,
+    )
+
+
+def render_ansible_inventory(inventory: PlatformInventory) -> dict[str, object]:
+    """Render static JSON accepted by Ansible's YAML inventory plugin."""
+    environment_groups: dict[str, object] = {}
+    environments = sorted(
+        {server.environment for server in inventory.servers},
+        key=lambda environment: environment.value,
+    )
+    for environment in environments:
+        environment_groups[_group_name("environment", environment)] = {
+            "hosts": _empty_host_entries(
+                server.resource_id.value
+                for server in inventory.servers
+                if server.environment == environment
+            )
+        }
+
+    application_groups: dict[str, object] = {}
+    for application in inventory.applications:
+        application_components = frozenset(application.component_ids)
+        application_hosts = sorted(
+            {
+                server_id.value
+                for component in inventory.components
+                if component.resource_id in application_components
+                for server_id in component.server_ids
+            }
+        )
+        application_groups[_group_name("application", application.resource_id)] = {
+            "hosts": _empty_host_entries(application_hosts)
+        }
+
+    component_groups = {
+        _group_name("component", component.resource_id): {
+            "hosts": _empty_host_entries(
+                server.value for server in component.server_ids
+            )
+        }
+        for component in inventory.components
+    }
+
+    logging_backend_hosts = _empty_host_entries(
+        stack.backend.server_id.value for stack in inventory.logging_stacks
+    )
+    logging_collector_hosts = _empty_host_entries(
+        server_id.value
+        for stack in inventory.logging_stacks
+        for server_id in stack.collectors.server_ids
+    )
+    domain_origin_hosts = _empty_host_entries(
+        domain.origin.server_id.value for domain in inventory.domains
+    )
+
+    return {
+        "all": {
+            "children": {
+                "ungrouped": {"hosts": {}},
+                "cloudfall_servers": {"hosts": _host_variables(inventory)},
+                "cloudfall_environments": {"children": environment_groups},
+                "cloudfall_applications": {"children": application_groups},
+                "cloudfall_components": {"children": component_groups},
+                "cloudfall_logging_backends": {"hosts": logging_backend_hosts},
+                "cloudfall_logging_collectors": {"hosts": logging_collector_hosts},
+                "cloudfall_domain_origins": {"hosts": domain_origin_hosts},
+            }
+        }
+    }
+
+
+def _host_variables(inventory: PlatformInventory) -> dict[str, object]:
+    applications_by_id = {
+        application.resource_id: application for application in inventory.applications
+    }
+    server_types_by_id = {
+        server_type.resource_id: server_type for server_type in inventory.server_types
+    }
+    return {
+        server.resource_id.value: {
+            "ansible_host": server.address.value,
+            "ansible_port": server.ssh_port.value,
+            "ansible_user": server.ssh_user.value,
+            "cloudfall_server_id": server.resource_id.value,
+            "cloudfall_hostname": server.hostname.value,
+            "cloudfall_environment": server.environment.value,
+            "cloudfall_lifecycle": server.lifecycle.value,
+            "cloudfall_labels": {
+                label.key.value: label.value for label in server.labels
+            },
+            "cloudfall_ssh_public_keys": [
+                key.as_dict()
+                for key in inventory.ssh_public_keys
+                if key.environment == server.environment and key.is_active
+            ],
+            **_host_infrastructure(server),
+            **_host_firewall(server, server_types_by_id[server.server_type_id]),
+            **_host_logging(server, inventory.logging_stacks, inventory.alert_rules),
+            "cloudfall_server_type": server_types_by_id[
+                server.server_type_id
+            ].as_dict(),
+            "cloudfall_applications": [
+                _host_application(application)
+                for application in inventory.applications
+                if _application_is_on_server(application, server, inventory)
+            ],
+            "cloudfall_components": [
+                _host_component(component, applications_by_id[component.application_id])
+                for component in inventory.components
+                if server.resource_id in component.server_ids
+            ],
+            "cloudfall_services": [
+                service.as_dict()
+                for service in inventory.services
+                if service.server_id == server.resource_id
+            ],
+            "cloudfall_domains": [
+                domain.as_dict()
+                for domain in inventory.domains
+                if domain.proxy.server_id == server.resource_id
+            ],
+            "cloudfall_origin_domains": [
+                domain.as_dict()
+                for domain in inventory.domains
+                if domain.origin.server_id == server.resource_id
+            ],
+        }
+        for server in inventory.servers
+    }
+
+
+def _host_infrastructure(server: ServerInventory) -> dict[str, object]:
+    variables: dict[str, object] = {}
+    if server.provider is not None:
+        variables["cloudfall_provider"] = server.provider.as_dict()
+    if server.network is not None:
+        variables["cloudfall_network"] = server.network.as_dict()
+    return variables
+
+
+def _host_firewall(
+    server: ServerInventory, server_type: ServerTypeInventory
+) -> dict[str, object]:
+    if server_type.firewall is None:
+        return {}
+    rules = firewall_rules_for_server(server_type.firewall, server.ssh_port)
+    return {
+        "cloudfall_firewall": {
+            "policy": server_type.firewall.policy.value,
+            "allowedInbound": [rule.as_dict() for rule in rules],
+        }
+    }
+
+
+def _host_logging(
+    server: ServerInventory,
+    logging_stacks: tuple[LoggingStackInventory, ...],
+    alert_rules: tuple[AlertRuleInventory, ...],
+) -> dict[str, object]:
+    variables: dict[str, object] = {}
+    backend = next(
+        (
+            stack
+            for stack in logging_stacks
+            if stack.backend.server_id == server.resource_id
+        ),
+        None,
+    )
+    collector = next(
+        (
+            stack
+            for stack in logging_stacks
+            if server.resource_id in stack.collectors.server_ids
+        ),
+        None,
+    )
+    if backend is not None:
+        variables["cloudfall_logging_backend"] = backend.as_dict()
+        variables["cloudfall_alert_rules"] = [
+            rule.as_dict()
+            for rule in sorted(
+                alert_rules, key=lambda rule: rule.resource_id.value
+            )
+            if rule.environment == backend.environment
+        ]
+    if collector is not None:
+        variables["cloudfall_logging_collector"] = collector.as_dict()
+    return variables
+
+
+def _host_application(application: ApplicationInventory) -> dict[str, object]:
+    return {
+        "id": application.resource_id.value,
+        "user": application.linux_user.value,
+        "approval": application.approval.value,
+    }
+
+
+def _host_component(
+    component: ComponentInventory, application: ApplicationInventory
+) -> dict[str, object]:
+    return {
+        "id": component.resource_id.value,
+        "application": component.application_id.value,
+        "application_user": application.linux_user.value,
+        "install_root": component.install_root.value,
+        "repository": component.repository.as_dict(),
+        "runtime": component.runtime.as_dict(),
+        "service": component.service.as_dict(),
+        "healthCheck": component.health_check.as_dict(),
+        "retainUntilCleanup": component.retain_until_cleanup,
+    }
+
+
+def _application_is_on_server(
+    application: ApplicationInventory,
+    server: ServerInventory,
+    inventory: PlatformInventory,
+) -> bool:
+    application_components = frozenset(application.component_ids)
+    return any(
+        component.resource_id in application_components
+        and server.resource_id in component.server_ids
+        for component in inventory.components
+    )
+
+
+def _empty_host_entries(hosts: Iterable[str]) -> dict[str, object]:
+    return {host: {} for host in hosts}
+
+
+def _group_name(prefix: str, resource_id: ResourceId) -> str:
+    return f"{prefix}_{resource_id.value.replace('-', '_')}"

cloudfall_engine/artifact.py

@@ -0,0 +1,258 @@
+"""Build verifiable release artifacts from component repositories."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import shutil
+import subprocess
+import tarfile
+import tempfile
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from cloudfall.validation import (
+    ConfigValidationError,
+    SchemaCatalog,
+    ValidationIssue,
+)
+from jsonschema.exceptions import ValidationError
+
+if TYPE_CHECKING:
+    from cloudfall.inventory import ComponentInventory, PlatformInventory
+
+_ARTIFACT_SCHEMA = "artifact.schema.json"
+_GIT_TIMEOUT_SECONDS = 600
+_SHORT_COMMIT_LENGTH = 7
+_GIT_REF_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/@^~-]{0,199}")
+_ERROR_COMPONENT_MISSING = "artifact_component_missing"
+_ERROR_SUBDIRECTORY_MISSING = "artifact_subdirectory_missing"
+_ERROR_REF_INVALID = "artifact_ref_invalid"
+_ERROR_GIT_MISSING = "artifact_git_missing"
+_ERROR_GIT_FAILED = "artifact_git_failed"
+_ERROR_GIT_TIMEOUT = "artifact_git_timeout"
+
+
+class ArtifactBuildError(RuntimeError):
+    """Fail-fast artifact build error with a stable machine-readable code."""
+
+    def __init__(self, code: str, message: str) -> None:
+        """Record the failure code and human-readable detail."""
+        self.code = code
+        self.detail = message
+        super().__init__(f"{code}: {message}")
+
+    def as_dict(self) -> dict[str, object]:
+        """Serialize the error envelope for system boundaries."""
+        return {
+            "status": "error",
+            "error": {"code": self.code, "message": self.detail},
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class BuiltArtifact:
+    """One built and hashed release artifact."""
+
+    component_id: str
+    release: str
+    git_ref: str
+    git_commit: str
+    built_at: str
+    archive_path: Path
+    metadata_path: Path
+    archive_sha256: str
+    size_bytes: int
+
+    def as_dict(self) -> dict[str, object]:
+        """Serialize the build result for CLI consumers."""
+        return {
+            "status": "ok",
+            "component": self.component_id,
+            "release": self.release,
+            "gitRef": self.git_ref,
+            "gitCommit": self.git_commit,
+            "builtAt": self.built_at,
+            "archive": str(self.archive_path),
+            "metadata": str(self.metadata_path),
+            "archiveSha256": self.archive_sha256,
+            "sizeBytes": self.size_bytes,
+        }
+
+
+def build_artifact(
+    inventory: PlatformInventory,
+    component_id: str,
+    git_ref: str,
+    output_directory: Path,
+    schema_directory: Path,
+) -> BuiltArtifact:
+    """Clone the component repository at a ref and package a release."""
+    component = _component(inventory, component_id)
+    _validate_ref(git_ref)
+    built_at = datetime.now(tz=UTC)
+    built_at_text = built_at.strftime("%Y-%m-%dT%H:%M:%SZ")
+    with tempfile.TemporaryDirectory(prefix="cloudfall-artifact-") as workdir:
+        checkout = Path(workdir) / "source"
+        _run_git("clone", "--quiet", component.repository.url.value, str(checkout))
+        # A fresh clone has local branches only for the default branch, and
+        # bare branch names trigger git's remote-branch DWIM, which conflicts
+        # with --detach. Resolving the ref to a commit first supports
+        # branches, tags, and commits uniformly.
+        commit = _resolve_ref_commit(checkout, git_ref)
+        _run_git(
+            "-C", str(checkout), "checkout", "--quiet", "--detach", commit, "--"
+        )
+        source_root = checkout
+        if component.repository.subdirectory is not None:
+            source_root = checkout / component.repository.subdirectory
+            if not source_root.is_dir():
+                detail = (
+                    "declared repository subdirectory does not exist: "
+                    f"{component.repository.subdirectory}"
+                )
+                raise ArtifactBuildError(
+                    _ERROR_SUBDIRECTORY_MISSING, detail
+                )
+
+        release = (
+            f"{built_at.strftime('%Y%m%dT%H%M%SZ')}"
+            f"-{commit[:_SHORT_COMMIT_LENGTH]}"
+        )
+        component_directory = output_directory / component_id
+        component_directory.mkdir(parents=True, exist_ok=True)
+        archive_path = component_directory / f"{release}.tar.gz"
+        _write_archive(source_root, archive_path)
+
+    digest = hashlib.sha256(archive_path.read_bytes()).hexdigest()
+    size_bytes = archive_path.stat().st_size
+    metadata = {
+        "apiVersion": "cloudfall/v1",
+        "kind": "Artifact",
+        "metadata": {
+            "id": component_id,
+            "description": f"Release artifact for {component_id}",
+        },
+        "spec": {
+            "component": component_id,
+            "release": release,
+            "gitRef": git_ref,
+            "gitCommit": commit,
+            "builtAt": built_at_text,
+            "archive": archive_path.name,
+            "archiveSha256": digest,
+            "sizeBytes": size_bytes,
+        },
+    }
+    catalog = SchemaCatalog(schema_directory)
+    try:
+        catalog.validate_named(_ARTIFACT_SCHEMA, metadata)
+    except ValidationError as error:
+        issue = ValidationIssue(
+            code="artifact_metadata_invalid",
+            message=error.message,
+        )
+        raise ConfigValidationError(issue) from error
+    metadata_path = archive_path.with_suffix("").with_suffix(".json")
+    metadata_path.write_text(
+        f"{json.dumps(metadata, indent=2, sort_keys=True)}\n",
+        encoding="utf-8",
+    )
+    return BuiltArtifact(
+        component_id=component_id,
+        release=release,
+        git_ref=git_ref,
+        git_commit=commit,
+        built_at=built_at_text,
+        archive_path=archive_path,
+        metadata_path=metadata_path,
+        archive_sha256=digest,
+        size_bytes=size_bytes,
+    )
+
+
+def _component(
+    inventory: PlatformInventory, component_id: str
+) -> ComponentInventory:
+    component = next(
+        (
+            candidate
+            for candidate in inventory.components
+            if candidate.resource_id.value == component_id
+        ),
+        None,
+    )
+    if component is None:
+        detail = f"component does not exist: {component_id}"
+        raise ArtifactBuildError(_ERROR_COMPONENT_MISSING, detail)
+    return component
+
+
+def _validate_ref(git_ref: str) -> None:
+    if not _GIT_REF_PATTERN.fullmatch(git_ref):
+        detail = f"git ref contains unsupported characters: {git_ref!r}"
+        raise ArtifactBuildError(_ERROR_REF_INVALID, detail)
+
+
+def _resolve_ref_commit(checkout: Path, git_ref: str) -> str:
+    candidates = (git_ref, f"origin/{git_ref}")
+    for candidate in candidates:
+        try:
+            return _run_git(
+                "-C",
+                str(checkout),
+                "rev-parse",
+                "--verify",
+                "--quiet",
+                "--end-of-options",
+                f"{candidate}^{{commit}}",
+            ).strip()
+        except ArtifactBuildError as error:
+            if error.code != _ERROR_GIT_FAILED:
+                raise
+    detail = f"git ref does not resolve to a commit: {git_ref!r}"
+    raise ArtifactBuildError(_ERROR_GIT_FAILED, detail)
+
+
+def _run_git(*arguments: str) -> str:
+    binary = shutil.which("git")
+    if binary is None:
+        detail = "git is not installed on the build host"
+        raise ArtifactBuildError(_ERROR_GIT_MISSING, detail)
+    try:
+        completed = subprocess.run(  # noqa: S603 - fixed binary, validated args.
+            [binary, *arguments],
+            capture_output=True,
+            check=True,
+            text=True,
+            timeout=_GIT_TIMEOUT_SECONDS,
+        )
+    except subprocess.CalledProcessError as error:
+        detail = (
+            f"git {' '.join(arguments[:2])} failed: {error.stderr.strip()}"
+        )
+        raise ArtifactBuildError(_ERROR_GIT_FAILED, detail) from error
+    except subprocess.TimeoutExpired as error:
+        detail = (
+            f"git {' '.join(arguments[:2])} exceeded "
+            f"{_GIT_TIMEOUT_SECONDS} seconds"
+        )
+        raise ArtifactBuildError(_ERROR_GIT_TIMEOUT, detail) from error
+    return completed.stdout
+
+
+def _write_archive(source_root: Path, archive_path: Path) -> None:
+    def _skip_git(info: tarfile.TarInfo) -> tarfile.TarInfo | None:
+        if ".git" in Path(info.name).parts:
+            return None
+        info.uid = 0
+        info.gid = 0
+        info.uname = "root"
+        info.gname = "root"
+        return info
+
+    with tarfile.open(archive_path, "w:gz") as archive:
+        archive.add(source_root, arcname=".", filter=_skip_git)