cloudfall 0.2.0__py3-none-any.whl

cloudfall/render_api.py

@@ -0,0 +1,338 @@
+"""Fetch live Render services and synthesize an importable blueprint.
+
+The Render REST API is read through a small injectable client and adapted
+into the exact mapping shape the blueprint importer already consumes, so
+both entry points share one mapping pipeline and one gap report. Secret
+material (connection strings fetched for matching) is compared in memory
+and never written anywhere.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.request
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Protocol, cast
+
+from cloudfall.importer import (
+    ImportGap,
+    RenderImportError,
+    import_render_mapping,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Mapping
+    from pathlib import Path
+
+    from cloudfall.importer import ImportTargets, RenderImportResult
+
+_ERROR_API_UNREACHABLE = "render_api_unreachable"
+_ERROR_API_INVALID = "render_api_invalid"
+_PAGE_LIMIT = 100
+_SERVICE_TYPES: Mapping[str, str] = {
+    "web_service": "web",
+    "background_worker": "worker",
+    "private_service": "pserv",
+    "cron_job": "cron",
+    "static_site": "static",
+    "keyvalue": "keyvalue",
+}
+_CATEGORY_ACTION = "action-required"
+_CATEGORY_ASSUMPTION = "assumption"
+
+
+class RenderApiClient(Protocol):
+    """Read-only view of one Render workspace."""
+
+    def list_services(self) -> tuple[Mapping[str, object], ...]:
+        """Return every service object in the workspace."""
+        ...
+
+    def list_env_vars(
+        self, service_id: str
+    ) -> tuple[Mapping[str, object], ...]:
+        """Return every environment variable of one service."""
+        ...
+
+    def list_postgres(self) -> tuple[Mapping[str, object], ...]:
+        """Return every managed PostgreSQL instance."""
+        ...
+
+    def postgres_connection_strings(self, postgres_id: str) -> tuple[str, ...]:
+        """Return the connection strings of one managed PostgreSQL."""
+        ...
+
+
+@dataclass(frozen=True, slots=True)
+class HttpRenderApiClient:
+    """Bearer-authenticated client for the Render REST API."""
+
+    api_key: str
+    base_url: str = "https://api.render.com/v1"
+    timeout_seconds: float = 30.0
+    transport: Callable[[str, str], object] | None = None
+
+    def list_services(self) -> tuple[Mapping[str, object], ...]:
+        """Return every service object in the workspace."""
+        return self._paginate("/services", "service")
+
+    def list_env_vars(
+        self, service_id: str
+    ) -> tuple[Mapping[str, object], ...]:
+        """Return every environment variable of one service."""
+        return self._paginate(f"/services/{service_id}/env-vars", "envVar")
+
+    def list_postgres(self) -> tuple[Mapping[str, object], ...]:
+        """Return every managed PostgreSQL instance."""
+        return self._paginate("/postgres", "postgres")
+
+    def postgres_connection_strings(self, postgres_id: str) -> tuple[str, ...]:
+        """Return the connection strings of one managed PostgreSQL."""
+        payload = self._get(f"/postgres/{postgres_id}/connection-info")
+        info = _require_mapping(payload, "postgres connection info")
+        return tuple(
+            value
+            for key in ("internalConnectionString", "externalConnectionString")
+            if isinstance((value := info.get(key)), str) and value
+        )
+
+    def _paginate(
+        self, path: str, envelope: str
+    ) -> tuple[Mapping[str, object], ...]:
+        items: list[Mapping[str, object]] = []
+        cursor: str | None = None
+        while True:
+            query = f"?limit={_PAGE_LIMIT}"
+            if cursor is not None:
+                query += f"&cursor={cursor}"
+            payload = self._get(f"{path}{query}")
+            if not isinstance(payload, list):
+                message = f"Render API returned a non-list page for {path}"
+                raise RenderImportError(_ERROR_API_INVALID, message)
+            rows = cast("list[object]", payload)
+            for raw_row in rows:
+                row = _require_mapping(raw_row, f"{envelope} page row")
+                items.append(_require_mapping(row.get(envelope), envelope))
+                raw_cursor = row.get("cursor")
+                if isinstance(raw_cursor, str):
+                    cursor = raw_cursor
+            if len(rows) < _PAGE_LIMIT:
+                return tuple(items)
+
+    def _get(self, path: str) -> object:
+        url = f"{self.base_url}{path}"
+        if self.transport is not None:
+            return self.transport(url, self.api_key)
+        request = urllib.request.Request(  # noqa: S310 - fixed https base
+            url,
+            headers={
+                "Authorization": f"Bearer {self.api_key}",
+                "Accept": "application/json",
+            },
+        )
+        try:
+            with urllib.request.urlopen(  # noqa: S310 - fixed https base
+                request, timeout=self.timeout_seconds
+            ) as response:
+                body = response.read()
+        except OSError as error:
+            message = f"Render API unreachable: {url}: {error}"
+            raise RenderImportError(_ERROR_API_UNREACHABLE, message) from error
+        try:
+            return cast("object", json.loads(body.decode("utf-8")))
+        except json.JSONDecodeError as error:
+            message = f"Render API returned invalid JSON for {path}"
+            raise RenderImportError(_ERROR_API_INVALID, message) from error
+
+
+@dataclass(slots=True)
+class _Synthesis:
+    """Accumulates the blueprint and API-side gap notes."""
+
+    services: list[dict[str, object]] = field(default_factory=list)
+    databases: list[dict[str, object]] = field(default_factory=list)
+    gaps: list[ImportGap] = field(default_factory=list)
+
+    def gap(self, category: str, subject: str, detail: str) -> None:
+        self.gaps.append(
+            ImportGap(category=category, subject=subject, detail=detail)
+        )
+
+
+def synthesize_blueprint(
+    client: RenderApiClient,
+) -> tuple[dict[str, object], tuple[ImportGap, ...]]:
+    """Fetch the live workspace and shape it like a Render blueprint."""
+    synthesis = _Synthesis()
+    connection_index = _database_connection_index(client, synthesis)
+    for service in client.list_services():
+        _synthesize_service(client, service, connection_index, synthesis)
+    blueprint: dict[str, object] = {"services": synthesis.services}
+    if synthesis.databases:
+        blueprint["databases"] = synthesis.databases
+    return blueprint, tuple(synthesis.gaps)
+
+
+def read_api_key(path: Path) -> str:
+    """Read the Render API key from a controller-side file."""
+    code = "render_api_key_missing"
+    if not path.is_file():
+        message = f"Render API key file does not exist: {path}"
+        raise RenderImportError(code, message)
+    api_key = path.read_text(encoding="utf-8").strip()
+    if not api_key:
+        message = f"Render API key file is empty: {path}"
+        raise RenderImportError(code, message)
+    return api_key
+
+
+def import_render_api(
+    client: RenderApiClient,
+    targets: ImportTargets,
+    schema_directory: Path,
+) -> RenderImportResult:
+    """Map one live Render workspace onto validated Cloudfall state files."""
+    blueprint, gaps = synthesize_blueprint(client)
+    return import_render_mapping(blueprint, targets, schema_directory, gaps)
+
+
+def _database_connection_index(
+    client: RenderApiClient, synthesis: _Synthesis
+) -> dict[str, str]:
+    """Map known connection strings onto blueprint database names."""
+    index: dict[str, str] = {}
+    for postgres in client.list_postgres():
+        raw_id = postgres.get("id")
+        name = str(postgres.get("name", "database"))
+        database: dict[str, object] = {"name": name}
+        database_name = postgres.get("databaseName")
+        if isinstance(database_name, str) and database_name:
+            database["databaseName"] = database_name
+        version = postgres.get("version")
+        if isinstance(version, str) and version:
+            database["postgresMajorVersion"] = version.split(".")[0]
+        if postgres.get("databaseUser") is not None:
+            database["user"] = postgres.get("databaseUser")
+        synthesis.databases.append(database)
+        if not isinstance(raw_id, str) or not raw_id:
+            synthesis.gap(
+                _CATEGORY_ACTION,
+                name,
+                "managed PostgreSQL has no id; connection strings in "
+                "environment variables cannot be rewritten automatically",
+            )
+            continue
+        for connection in client.postgres_connection_strings(raw_id):
+            index[connection] = name
+    return index
+
+
+def _synthesize_service(
+    client: RenderApiClient,
+    service: Mapping[str, object],
+    connection_index: Mapping[str, str],
+    synthesis: _Synthesis,
+) -> None:
+    name = str(service.get("name", "unnamed"))
+    raw_type = str(service.get("type", ""))
+    blueprint_type = _SERVICE_TYPES.get(raw_type, raw_type)
+    entry = _service_entry(service, name, blueprint_type)
+    if service.get("suspended") == "suspended":
+        synthesis.gap(
+            _CATEGORY_ASSUMPTION,
+            name,
+            "service is suspended on Render; imported anyway",
+        )
+    raw_id = service.get("id")
+    if isinstance(raw_id, str) and raw_id and blueprint_type not in (
+        "static",
+        "keyvalue",
+    ):
+        entry["envVars"] = _synthesize_env_vars(
+            client.list_env_vars(raw_id), name, connection_index, synthesis
+        )
+    synthesis.services.append(entry)
+
+
+def _service_entry(
+    service: Mapping[str, object],
+    name: str,
+    blueprint_type: str,
+) -> dict[str, object]:
+    details = _optional_mapping(service.get("serviceDetails"))
+    specifics = (
+        _optional_mapping(details.get("envSpecificDetails"))
+        if details is not None
+        else None
+    )
+    entry: dict[str, object] = {"type": blueprint_type, "name": name}
+    detail_keys = (("env", "runtime"), ("healthCheckPath", "healthCheckPath"),
+                   ("schedule", "schedule"))
+    if details is not None:
+        for source_key, target_key in detail_keys:
+            value = details.get(source_key)
+            if isinstance(value, str) and value:
+                entry[target_key] = value
+    for source_key in ("repo", "branch"):
+        value = service.get(source_key)
+        if isinstance(value, str) and value:
+            entry[source_key] = value
+    if specifics is not None:
+        for source_key in ("buildCommand", "startCommand"):
+            value = specifics.get(source_key)
+            if isinstance(value, str) and value:
+                entry[source_key] = value
+    return entry
+
+
+def _synthesize_env_vars(
+    env_vars: tuple[Mapping[str, object], ...],
+    name: str,
+    connection_index: Mapping[str, str],
+    synthesis: _Synthesis,
+) -> list[dict[str, object]]:
+    entries: list[dict[str, object]] = []
+    for env_var in env_vars:
+        key = env_var.get("key")
+        if not isinstance(key, str) or not key:
+            synthesis.gap(
+                _CATEGORY_ACTION,
+                name,
+                "the Render API returned an environment entry without a key",
+            )
+            continue
+        value = env_var.get("value")
+        if not isinstance(value, str):
+            entries.append({"key": key, "sync": False})
+            continue
+        database_name = connection_index.get(value)
+        if database_name is not None:
+            entries.append(
+                {
+                    "key": key,
+                    "fromDatabase": {
+                        "name": database_name,
+                        "property": "connectionString",
+                    },
+                }
+            )
+            continue
+        entries.append({"key": key, "value": value})
+    return entries
+
+
+def _require_mapping(value: object, concept: str) -> Mapping[str, object]:
+    if not isinstance(value, dict) or not all(
+        isinstance(key, str) for key in value
+    ):
+        message = f"Render API {concept} is not an object"
+        raise RenderImportError(_ERROR_API_INVALID, message)
+    return cast("Mapping[str, object]", value)
+
+
+def _optional_mapping(value: object) -> Mapping[str, object] | None:
+    if not isinstance(value, dict) or not all(
+        isinstance(key, str) for key in value
+    ):
+        return None
+    return cast("Mapping[str, object]", value)

cloudfall/resources.py

@@ -0,0 +1,67 @@
+"""Locate the schema catalog and execution engine that ship with Cloudfall.
+
+Cloudfall runs in two shapes. Installed as a wheel, the versioned JSON
+schemas and the Ansible engine travel inside the package as bundled data.
+Run from a source checkout, they live in the repository tree next to the
+packages. Every command-line default resolves through this module so a
+project never has to know which shape it is talking to.
+"""
+
+from __future__ import annotations
+
+from importlib import resources
+from pathlib import Path
+
+import cloudfall
+
+_SCHEMA_VERSION = "v1"
+_BUNDLE = "_bundled"
+
+
+class ResourceError(RuntimeError):
+    """Raised when neither a bundled nor a checkout resource exists."""
+
+
+def _package_directory(package: object) -> Path:
+    spec = getattr(package, "__file__", None)
+    if not isinstance(spec, str):
+        message = f"package has no filesystem location: {package!r}"
+        raise ResourceError(message)
+    return Path(spec).resolve().parent
+
+
+def _checkout_root() -> Path:
+    # sdk/src/cloudfall -> sdk/src -> sdk -> repository root
+    return _package_directory(cloudfall).parents[2]
+
+
+def select_resource(bundled: Path, checkout: Path, description: str) -> Path:
+    """Prefer the bundled copy, fall back to the checkout, else fail."""
+    if bundled.is_dir():
+        return bundled
+    if checkout.is_dir():
+        return checkout
+    message = (
+        f"{description} is missing: neither the bundled copy {bundled} nor "
+        f"the source checkout copy {checkout} exists"
+    )
+    raise ResourceError(message)
+
+
+def default_schema_directory() -> Path:
+    """Return the versioned schema directory Cloudfall ships with."""
+    bundled = (
+        Path(str(resources.files("cloudfall").joinpath(_BUNDLE, "schemas")))
+        / _SCHEMA_VERSION
+    )
+    checkout = _checkout_root() / "config" / "schemas" / _SCHEMA_VERSION
+    return select_resource(bundled, checkout, "schema directory")
+
+
+def default_engine_directory() -> Path:
+    """Return the engine directory holding the bundled Ansible contracts."""
+    bundled = Path(str(resources.files("cloudfall_engine").joinpath(_BUNDLE)))
+    checkout = _checkout_root() / "engine"
+    return select_resource(
+        bundled / "ansible", checkout / "ansible", "engine directory"
+    ).parent

cloudfall/secrets.py

@@ -0,0 +1,262 @@
+"""Resolve declared secret references into component environment files.
+
+Values live sops-encrypted in a private repository and are decrypted only
+on the controller, at render time, into a single 0600 environment file per
+component. State carries references, receipts carry hashes and key names;
+a secret value never appears in state, output envelopes, or the agent
+surface.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import shutil
+import subprocess
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, cast
+
+from cloudfall.inventory import PlatformInventory
+from cloudfall.validation import SchemaCatalog, validate_config
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
+    from cloudfall.domain import ResourceId
+    from cloudfall.inventory import ComponentInventory, SecretReference
+    from cloudfall.lifecycle import EngineContext
+
+_ERROR_SOPS_MISSING = "secrets_sops_missing"
+_ERROR_SOURCE_MISSING = "secrets_source_missing"
+_ERROR_SOURCE_INVALID = "secrets_source_invalid"
+_ERROR_COMPONENT_UNKNOWN = "secrets_component_unknown"
+_ERROR_DECRYPT_FAILED = "secrets_decrypt_failed"
+_ERROR_KEY_CONFLICT = "secrets_key_conflict"
+_RECEIPT_SCHEMA = "environment-receipt.schema.json"
+_KEY_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+_FILE_MODE = 0o600
+
+
+class SecretsError(RuntimeError):
+    """Structured secrets failure that never carries a value."""
+
+    def __init__(self, code: str, message: str) -> None:
+        """Capture a stable error code alongside the human message."""
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+    def as_dict(self) -> dict[str, object]:
+        """Serialize the failure for structured output."""
+        return {
+            "status": "error",
+            "error": {"code": self.code, "message": self.message},
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class SopsSecretProvider:
+    """Reads dotenv fragments sops-encrypted in a secrets directory.
+
+    The on-disk layout mirrors the reference itself:
+    ``<secrets_directory>/<environment><path>.env`` — for example the
+    reference ``{environment: production, path: /crm/backend}`` resolves
+    from ``secrets/production/crm/backend.env``.
+    """
+
+    secrets_directory: Path
+    decrypt: Callable[[Path], str] | None = None
+
+    def fetch(
+        self, reference: SecretReference
+    ) -> tuple[tuple[str, str], ...]:
+        """Decrypt one reference's fragment into ordered key pairs."""
+        source = self.source_path(reference)
+        if not source.is_file():
+            message = (
+                f"secret source does not exist: {source} (referenced as "
+                f"scope={reference.scope.value} path={reference.path.value})"
+            )
+            raise SecretsError(_ERROR_SOURCE_MISSING, message)
+        text = (
+            self.decrypt(source)
+            if self.decrypt is not None
+            else _sops_decrypt(source)
+        )
+        return _parse_dotenv(text, source)
+
+    def source_path(self, reference: SecretReference) -> Path:
+        """Return the encrypted file backing one reference."""
+        relative = reference.path.value.lstrip("/")
+        return (
+            self.secrets_directory
+            / reference.environment.value
+            / f"{relative}.env"
+        )
+
+
+def render_environment(
+    context: EngineContext,
+    component_id: ResourceId,
+    provider: SopsSecretProvider,
+    output_path: Path,
+    receipt_directory: Path | None = None,
+) -> dict[str, object]:
+    """Resolve a component's references into one 0600 environment file."""
+    state = validate_config(context.project_directory, context.schema_directory)
+    inventory = PlatformInventory.from_state(state)
+    component = _declared_component(inventory, component_id)
+    application = next(
+        application
+        for application in inventory.applications
+        if application.resource_id == component.application_id
+    )
+    references = (*application.secret_refs, *component.secret_refs)
+    declared = dict(component.environment)
+    merged: dict[str, str] = dict(declared)
+    for reference in references:
+        for key, value in provider.fetch(reference):
+            if key in declared:
+                message = (
+                    f"{key} is declared in the component's non-secret "
+                    "environment and also provided by the secret source "
+                    f"{provider.source_path(reference)}; a key must live "
+                    "in exactly one place"
+                )
+                raise SecretsError(_ERROR_KEY_CONFLICT, message)
+            merged[key] = value
+    content = "".join(f"{key}={value}\n" for key, value in merged.items())
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    descriptor = os.open(
+        output_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _FILE_MODE
+    )
+    with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+        handle.write(content)
+    output_path.chmod(_FILE_MODE)
+    digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
+    envelope: dict[str, object] = {
+        "status": "ok",
+        "component": component_id.value,
+        "environmentFile": str(output_path),
+        "sha256": digest,
+        "keys": sorted(merged),
+        "declaredKeys": sorted(declared),
+        "references": [reference.as_dict() for reference in references],
+    }
+    if receipt_directory is not None:
+        envelope["receipt"] = str(
+            _write_receipt(
+                context, component_id, digest, sorted(merged),
+                receipt_directory,
+            )
+        )
+    return envelope
+
+
+def _write_receipt(
+    context: EngineContext,
+    component_id: ResourceId,
+    digest: str,
+    keys: list[str],
+    receipt_directory: Path,
+) -> Path:
+    document: dict[str, object] = {
+        "apiVersion": "cloudfall/v1",
+        "kind": "EnvironmentReceipt",
+        "metadata": {
+            "id": component_id.value,
+            "description": "Rendered environment receipt",
+        },
+        "spec": {
+            "component": component_id.value,
+            "sha256": digest,
+            "renderedAt": datetime.now(UTC).isoformat(timespec="seconds"),
+            "keys": keys,
+        },
+    }
+    SchemaCatalog(context.schema_directory).validate_named(
+        _RECEIPT_SCHEMA, document
+    )
+    receipt_directory.mkdir(parents=True, exist_ok=True)
+    path = receipt_directory / f"{component_id.value}.json"
+    path.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8")
+    return path
+
+
+def load_environment_receipts(
+    receipt_directory: Path, schema_directory: Path
+) -> dict[str, str]:
+    """Load rendered-environment hashes by component id."""
+    if not receipt_directory.is_dir():
+        return {}
+    catalog = SchemaCatalog(schema_directory)
+    hashes: dict[str, str] = {}
+    for path in sorted(receipt_directory.glob("*.json")):
+        document = cast(
+            "dict[str, object]",
+            json.loads(path.read_text(encoding="utf-8")),
+        )
+        catalog.validate_named(_RECEIPT_SCHEMA, document)
+        spec = cast("dict[str, object]", document["spec"])
+        hashes[cast("str", spec["component"])] = cast("str", spec["sha256"])
+    return hashes
+
+
+def _declared_component(
+    inventory: PlatformInventory, component_id: ResourceId
+) -> ComponentInventory:
+    component = next(
+        (
+            candidate
+            for candidate in inventory.components
+            if candidate.resource_id == component_id
+        ),
+        None,
+    )
+    if component is None:
+        message = f"declared component does not exist: {component_id.value}"
+        raise SecretsError(_ERROR_COMPONENT_UNKNOWN, message)
+    return component
+
+
+def _sops_decrypt(source: Path) -> str:
+    binary = shutil.which("sops")
+    if binary is None:
+        message = (
+            "the sops binary is not installed; install sops and age, and "
+            "set SOPS_AGE_KEY_FILE to your age key"
+        )
+        raise SecretsError(_ERROR_SOPS_MISSING, message)
+    completed = subprocess.run(  # noqa: S603 - fixed binary, declared file
+        [binary, "--decrypt", str(source)],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if completed.returncode != 0:
+        message = (
+            f"sops could not decrypt {source}: "
+            f"{completed.stderr.strip()[:300]}"
+        )
+        raise SecretsError(_ERROR_DECRYPT_FAILED, message)
+    return completed.stdout
+
+
+def _parse_dotenv(text: str, source: Path) -> tuple[tuple[str, str], ...]:
+    pairs: list[tuple[str, str]] = []
+    for line_number, raw_line in enumerate(text.splitlines(), start=1):
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        key, separator, value = line.partition("=")
+        if not separator or not _KEY_PATTERN.fullmatch(key):
+            message = (
+                f"{source} line {line_number} is not a KEY=VALUE entry"
+            )
+            raise SecretsError(_ERROR_SOURCE_INVALID, message)
+        pairs.append((key, value))
+    return tuple(pairs)