boatyard 0.6.0.dev170__py3-none-any.whl

boatyard/score.py

@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+import os
+import re
+import shlex
+from contextlib import ExitStack
+from hashlib import sha256
+from importlib.resources import as_file, files
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Literal, NamedTuple, Self
+
+import yaml
+from bashrun.bash import bash
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    RootModel,
+    ValidationInfo,
+    field_validator,
+    model_validator,
+)
+
+from boatyard.config.config import (
+    ENV_NAME_PATTERN,
+    PLACEHOLDER,
+    Yaml,
+    is_list,
+    is_mapping,
+    ConfigFile,
+    WorkloadDocument,
+)
+from boatyard.yaml import load_yaml_model
+from boatyard.markers import MarkerContext, resolve_markers
+
+PATCH_TEMPLATES_DIR = "score_patch_templates"
+
+SECRETS_FILE = Path("stack/secrets.yaml")
+SECRETS_OUTPUT = Path("stack/generated/secrets.txt")
+
+STRING_FORM_FIELDS = ("state", "outputs", "volumes", "services", "manifests")
+
+Substrate = Literal["compose", "k8s"]
+
+
+class SecretsDeclaration(RootModel[dict[str, "SecretKey"]]):
+    @model_validator(mode="before")
+    @classmethod
+    def _coerce_string_specs(cls, value: Yaml) -> Yaml:
+        if value is None:
+            value = {}
+        if not is_mapping(value):
+            raise ValueError("must be a KEY: description mapping")
+        return {name: ({"description": spec} if isinstance(spec, str) else spec) for name, spec in value.items()}
+
+    @model_validator(mode="after")
+    def _require_all_caps_keys(self) -> Self:
+        for name in self.root:
+            if not ENV_NAME_PATTERN.match(name):
+                raise ValueError(f"{name!r} must be an ALL_CAPS key")
+        return self
+
+    def validate_consumed(self, consumed: set[str], rendering: Substrate, source: str) -> None:
+        undeclared = sorted(consumed - self.root.keys())
+        if undeclared:
+            raise ValueError(f"{source}: the {rendering} rendering consumes undeclared keys: " + ", ".join(undeclared))
+
+
+class SecretKey(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    description: str
+    mint: bool = False
+
+    @field_validator("description")
+    @classmethod
+    def _require_description(cls, value: str) -> str:
+        if not value.strip():
+            raise ValueError("description must be non-empty")
+        return value
+
+
+class AuthoredProvisioner(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    uri: str
+    type: str
+    description: str = ""
+    state: dict[str, Yaml] | None = None
+    outputs: dict[str, Yaml] | None = None
+    volumes: dict[str, Yaml] | None = None
+    services: dict[str, Yaml] | None = None
+    manifests: list[Yaml] | None = None
+
+    @field_validator(*STRING_FORM_FIELDS, mode="before")
+    @classmethod
+    def _reject_string_form(cls, value: Yaml, info: ValidationInfo) -> Yaml:
+        if not isinstance(value, str):
+            return value
+
+        shape = "list" if info.field_name == "manifests" else "mapping"
+        raise ValueError(f"{info.field_name} is in score's compiled string form; a {shape} is required")
+
+
+class LiteralBlockDumper(yaml.SafeDumper):
+    pass
+
+
+class LoadedWorkload(NamedTuple):
+    file_name: str
+    raw: Yaml
+    document: WorkloadDocument
+
+
+class ToolState(BaseModel):
+    # Type only the workloads shape of the tool-owned state file
+    model_config = ConfigDict(extra="allow")
+
+    workloads: dict[str, dict[str, Yaml]] = Field(default_factory=dict)
+
+
+def _represent_multiline(dumper: yaml.SafeDumper, value: str) -> yaml.ScalarNode:
+    return yaml.ScalarNode("tag:yaml.org,2002:str", value, style="|" if "\n" in value else None)
+
+
+LiteralBlockDumper.add_representer(str, _represent_multiline)
+
+
+def render_score(
+    config: ConfigFile,
+    substrate: Substrate,
+    workloads: list[LoadedWorkload],
+    rendering_vars: dict[str, str],
+    templates: list[str],
+    output: Path,
+    images: dict[str, str],
+    service_shas: dict[str, str],
+) -> None:
+    # Derive the per-substrate names and create the output parent
+    root = config.root
+    score_dir = root / config.score_dir
+    tool = f"score-{substrate}"
+    rendering = substrate
+    # Run both tools in the output directory because both write their state dir into the cwd
+    working_dir = output.parent
+    project_name = None if substrate == "k8s" else config.project_name
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    with (
+        as_file(files("boatyard") / PATCH_TEMPLATES_DIR) as builtin_templates,
+        ExitStack() as stack,
+    ):
+        # Collect the resource types the workloads reference
+        referenced = {resource.type for workload in workloads for resource in workload.document.resources.values()}
+
+        # Select the rendering's provisioner files that declare a referenced type
+        selected: list[tuple[str, list[Yaml]]] = []
+        rendering_dir = score_dir / rendering
+        if rendering_dir.is_dir():
+            for path in sorted(rendering_dir.glob("*.yaml")):
+                entries = yaml.safe_load(path.read_text(encoding="utf-8"))
+                if is_list(entries) and any(is_mapping(entry) and entry.get("type") in referenced for entry in entries):
+                    selected.append((f"{rendering}/{path.name}", entries))
+
+        # Compile the selected provisioners into a per-run tempdir under score_dir
+        paths: list[Path] = []
+        directory = Path(stack.enter_context(TemporaryDirectory(prefix="compiled-provisioners-", dir=score_dir)))
+        for name, entries in selected:
+            context = MarkerContext(root=root, images=images, source=name)
+            compiled_entries: list[Yaml] = []
+            for entry in entries:
+                resolved = resolve_markers(entry, context)
+                AuthoredProvisioner.model_validate(resolved)
+                if is_mapping(resolved):
+                    compiled_entries.append({
+                        key: dump_yaml(value) if key in STRING_FORM_FIELDS and value is not None else value
+                        for key, value in resolved.items()
+                    })
+            target = directory / name
+            target.parent.mkdir(parents=True, exist_ok=True)
+            write_file(target, dump_yaml(compiled_entries))
+            paths.append(target)
+
+        # Build the tool flags and run init
+        provisioner_flags = "".join(
+            f" --provisioners {shlex.quote(Path(os.path.relpath(path, working_dir)).as_posix())}" for path in paths
+        )
+        template_flags = "".join(
+            f" --patch-templates {shlex.quote(Path(os.path.relpath(builtin_templates / name, working_dir)).as_posix())}"
+            for name in templates
+        )
+        bash(
+            f"{tool} init{' --project ' + project_name if project_name is not None else ''}"
+            f" --no-sample{provisioner_flags}{template_flags}",
+            cwd=working_dir,
+        )
+
+        # Pre-seed each workload's instance suffix so generated names are deterministic
+        if substrate == "k8s":
+            state_path = output.parent / f".{tool}" / "state.yaml"
+            state = load_yaml_model(ToolState, state_path)
+            for workload in workloads:
+                workload_name = Path(workload.file_name).stem
+                state.workloads.setdefault(workload_name, {})["instance_suffix"] = (
+                    "-" + sha256(workload_name.encode()).hexdigest()[:10]
+                )
+            write_file(state_path, dump_yaml(state.model_dump()))
+
+        # Expand each workload's ${NAME} slots into per-run copies
+        copies_directory = Path(stack.enter_context(TemporaryDirectory()))
+        # List the variable sources lowest-priority first so later ones overwrite earlier ones
+        variable_values = {
+            variable: value
+            for source in (images, service_shas, rendering_vars)
+            for variable, value in source.items()
+            if value
+        }
+
+        for workload in workloads:
+            expanded = dump_yaml(workload.raw)
+            for variable, value in variable_values.items():
+                expanded = expanded.replace(f"${{{variable}}}", value)
+
+            # Refuse slots that no source fills
+            leftover = PLACEHOLDER.search(expanded)
+            if leftover is not None:
+                raise ValueError(
+                    f"{workload.file_name}: ${{{leftover.group(1)}}} is set in none of the rendering vars, "
+                    "the service shas, or the image lock"
+                )
+
+            (copies_directory / workload.file_name).write_text(expanded, "utf-8")
+
+        # Run generate over the expanded copies
+        rendered = " ".join((copies_directory / workload.file_name).as_posix() for workload in workloads)
+        bash(f"{tool} generate {rendered} --output {output.as_posix()}", cwd=working_dir)
+
+
+def load_workloads(score_dir: Path, names: list[str], lens: type[WorkloadDocument]) -> list[LoadedWorkload]:
+    loaded: list[LoadedWorkload] = []
+    for name in names:
+        raw = yaml.safe_load((score_dir / name).read_text(encoding="utf-8"))
+        loaded.append(LoadedWorkload(name, raw, lens.parse(raw, name)))
+    return loaded
+
+
+def dump_yaml(value: Yaml) -> str:
+    return yaml.dump(
+        value,
+        Dumper=LiteralBlockDumper,
+        default_flow_style=False,
+        sort_keys=False,
+        width=1_000_000,
+        allow_unicode=True,
+    )
+
+
+def write_file(path: Path, text: str) -> None:
+    with path.open("w", encoding="utf-8", newline="") as handle:
+        handle.write(text)
+
+    path.chmod(0o644)
+
+
+def split_documents(text: str) -> list[str]:
+    return [block for block in re.split(r"(?m)^---$\n", text) if block.strip()]

boatyard/score_compose.py

@@ -0,0 +1,274 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import Literal, Self
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    field_validator,
+    model_validator,
+)
+
+from boatyard.config.config import (
+    COMPOSE_SLOT,
+    ComposeWorkloadDocument,
+    WorkloadDocument,
+    Yaml,
+    compose_bundle_output_path,
+    double_dollars,
+    env_lock_slot,
+    ConfigFile,
+)
+from boatyard.config.lock import LOCK_PATH, LockFile
+from boatyard.yaml import load_yaml_model
+from boatyard.score import (
+    SECRETS_FILE,
+    SecretsDeclaration,
+    dump_yaml,
+    load_workloads,
+    render_score,
+    write_file,
+)
+from boatyard.config.values import ValuesFile
+from boatyard.paths import VALUES_FILE
+
+TOOL = "score-compose"
+
+BARRIER_SERVICE = "wait-for-resources"
+HEALTHCHECK_TEMPLATE = "probe-adapter-healthchecks.tpl"
+RESTART_TEMPLATE = "restart.tpl"
+GPU_COMPOSE_TEMPLATES = {"cuda": "gpu-cuda.tpl", "rocm": "gpu-rocm.tpl"}
+
+
+class ComposeDocument(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    services: dict[str, ComposeService] = Field(default_factory=dict)
+    configs: dict[str, ComposeConfig] | None = None
+    networks: dict[str, Yaml] | None = None
+
+    @model_validator(mode="after")
+    def _reject_colliding_bind_basenames(self) -> Self:
+        seen: dict[str, str] = {}
+        for service in self.services.values():
+            for volume in service.bind_volumes:
+                source = volume.source
+                known = seen.setdefault(Path(source).name, source)
+                if known != source:
+                    raise ValueError(f"bind sources {known} and {source} collide on config name {Path(source).name}")
+        return self
+
+    def dump(self) -> Yaml:
+        return self.model_dump(exclude_none=True)
+
+
+class ComposeService(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    volumes: list[BindVolume | OtherVolume] | None = None
+    depends_on: dict[str, Yaml] | list[str] | None = None
+    configs: list[ComposeConfigReference] | None = None
+    image: str | None = None
+    networks: dict[str, Yaml] | None = None
+
+    @property
+    def bind_volumes(self) -> list[BindVolume]:
+        return [volume for volume in self.volumes or [] if isinstance(volume, BindVolume)]
+
+    def strip_bind_volumes(self) -> list[BindVolume]:
+        binds = self.bind_volumes
+        self.volumes = [volume for volume in self.volumes or [] if not isinstance(volume, BindVolume)] or None
+        return binds
+
+
+class ComposeConfigReference(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    source: str
+    target: str
+
+
+class BindVolume(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    type: Literal["bind"]
+    source: str
+    target: str
+
+    @field_validator("source")
+    @classmethod
+    def _require_relative(cls, value: str) -> str:
+        if Path(value).is_absolute():
+            raise ValueError(f"bind source {value} must be relative")
+        return value
+
+
+class OtherVolume(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    type: str
+
+    @model_validator(mode="after")
+    def _reject_bind(self) -> Self:
+        if self.type == "bind":
+            raise ValueError("a bind volume is missing its source or target")
+        return self
+
+
+class ComposeConfig(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    file: str | None = None
+    content: str | None = None
+
+
+def generate_compose(
+    config: ConfigFile, lock: LockFile, service_shas: dict[str, str], secrets: SecretsDeclaration
+) -> list[Path]:
+    score_dir = config.root / config.score_dir
+    state_prefix = f".{TOOL}/"
+    values_path = config.root / VALUES_FILE
+    platform = ValuesFile.load(values_path, config).platform if values_path.is_file() else None
+    images = lock.manifest_references(platform)
+
+    if "ALPINE_IMAGE" not in images:
+        raise ValueError(f"{LOCK_PATH.as_posix()}: the image lock must carry the ALPINE_IMAGE entry")
+
+    outputs: list[Path] = []
+    consumed: set[str] = set()
+    for name, entry in config.local_service_entries():
+        workloads = load_workloads(score_dir, entry.workloads, ComposeWorkloadDocument)
+
+        output_path = compose_bundle_output_path(config.root, name)
+        tool_state = output_path.parent / f".{TOOL}"
+
+        # Render the bundle from a clean state dir
+        shutil.rmtree(tool_state, ignore_errors=True)
+        render_score(
+            config,
+            substrate="compose",
+            workloads=workloads,
+            rendering_vars=entry.vars,
+            templates=[
+                HEALTHCHECK_TEMPLATE,
+                RESTART_TEMPLATE,
+                *(
+                    template
+                    for variant, template in GPU_COMPOSE_TEMPLATES.items()
+                    if variant == entry.vars.get("GPU_VARIANT", "")
+                ),
+            ],
+            output=output_path,
+            images=images,
+            service_shas=service_shas,
+        )
+
+        # Parse the rendering into the typed document
+        document = load_yaml_model(ComposeDocument, output_path)
+
+        # Attribute rendered container services to their owning workloads
+        services_map: dict[str, tuple[ComposeService, WorkloadDocument | None, str]] = {
+            (service_name := f"{Path(workload.file_name).stem}-{container_name}"): (
+                document.services[service_name],
+                workload.document,
+                container_name,
+            )
+            for workload in workloads
+            for container_name in workload.document.containers
+        }
+
+        # Carry workload-less services through under their rendered names
+        for service_name, service in document.services.items():
+            if service_name in services_map:
+                continue
+
+            # Pin the barrier image to the lock slot
+            if service_name == BARRIER_SERVICE:
+                barrier_service = document.services[BARRIER_SERVICE]
+                barrier_service.image = "${ALPINE_IMAGE:?err}"
+                services_map[BARRIER_SERVICE] = (barrier_service, None, BARRIER_SERVICE)
+                continue
+
+            services_map[service_name] = (service, None, service_name)
+
+        # Refuse a collapsed name claimed twice
+        collapsed_names = [collapsed_name for _service, _workload, collapsed_name in services_map.values()]
+        if len(collapsed_names) != len(set(collapsed_names)):
+            duplicates = sorted({collapsed for collapsed in collapsed_names if collapsed_names.count(collapsed) > 1})
+            raise ValueError(f"the service names {duplicates} are each claimed by more than one rendered service")
+
+        # Namespace every service under the entry with its bare name as a network alias
+        network = f"{name}-net"
+        collapsed = set(collapsed_names)
+        document.services = {}
+        document.networks = {network: {}}
+        for service, workload, collapsed_name in services_map.values():
+            document.services[f"{name}-{collapsed_name}"] = service
+            service.networks = {network: {"aliases": [collapsed_name]}}
+
+            # Namespace dependency keys that name rendered services of this bundle
+            depends_on = service.depends_on
+            if isinstance(depends_on, dict):
+                service.depends_on = {
+                    f"{name}-{dependency}" if dependency in collapsed else dependency: condition
+                    for dependency, condition in depends_on.items()
+                }
+            elif isinstance(depends_on, list):
+                service.depends_on = [
+                    f"{name}-{dependency}" if dependency in collapsed else dependency for dependency in depends_on
+                ]
+
+            # Inline every bind volume as a top-level configs entry
+            bind_volumes = service.strip_bind_volumes()
+            if not bind_volumes:
+                continue
+
+            # Ensure the configs dict exists and the service has a configs list
+            if document.configs is None:
+                document.configs = {}
+            if service.configs is None:
+                service.configs = []
+
+            for bind_volume in bind_volumes:
+                # Resolve state-dir sources against the tool's working directory, others against root
+                path = (
+                    output_path.parent / bind_volume.source
+                    if bind_volume.source.startswith(state_prefix)
+                    else config.root / bind_volume.source
+                )
+
+                # Double every $ except on targets the workload declared consumer-interpolated
+                text = path.read_text(encoding="utf-8")
+                if workload and bind_volume.target not in workload.consumer_interpolated_targets:
+                    text = double_dollars(text)
+
+                document.configs[f"{name}-{path.name}"] = ComposeConfig(content=text)
+                service.configs.append(ComposeConfigReference(source=f"{name}-{path.name}", target=bind_volume.target))
+
+        # Scan consumed secrets and namespace the lock-key slots in one pass
+        text = dump_yaml(document.dump())
+        namespaced: list[str] = []
+        cursor = 0
+        for match in COMPOSE_SLOT.finditer(text):
+            namespaced.append(text[cursor : match.start()])
+            cursor = match.end()
+            if len(match.group(1)) % 2 == 1:
+                slot = match.group(2)
+                if slot in images:
+                    namespaced.append(f"{match.group(1)}{{{env_lock_slot(name, slot)}{match.group(3) or ''}}}")
+                    continue
+                consumed.add(slot)
+            namespaced.append(match.group(0))
+        namespaced.append(text[cursor:])
+        write_file(output_path, "".join(namespaced))
+        outputs.append(output_path)
+
+        # Remove the entry's compose state dir
+        shutil.rmtree(tool_state, ignore_errors=True)
+
+    secrets.validate_consumed(consumed, "compose", SECRETS_FILE.as_posix())
+
+    return outputs

boatyard/score_k8s.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import yaml
+from bashrun.bash import bash_output
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+)
+
+from boatyard.config.config import (
+    K8sConfig,
+    WorkloadDocument,
+    Yaml,
+    ConfigFile,
+    chart_lock_key,
+    is_list,
+    is_mapping,
+)
+from boatyard.config.lock import LockFile
+from boatyard.markers import MarkerContext, resolve_markers
+from boatyard.score import (
+    SECRETS_FILE,
+    SecretsDeclaration,
+    dump_yaml,
+    load_workloads,
+    render_score,
+    split_documents,
+    write_file,
+)
+
+GPU_POD_TEMPLATE = "gpu-pod.tpl"
+SCALE_TO_ZERO_TEMPLATE = "scale-to-zero.tpl"
+
+STORAGE_CLASS_VAR = "SCORE_STORAGE_CLASS"
+CLOUD_STORAGE_CLASS = "hcloud-volumes"
+LOCAL_STORAGE_CLASS = "local-path"
+
+
+class ManifestMetadata(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    namespace: str = ""
+    name: str = ""
+
+
+class ManifestDocument(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    kind: str = ""
+    metadata: ManifestMetadata = Field(default_factory=ManifestMetadata)
+
+
+def generate_k8s(
+    config: ConfigFile,
+    k8s: K8sConfig,
+    lock: LockFile,
+    service_shas: dict[str, str],
+    local: bool,
+    secrets: SecretsDeclaration,
+) -> Path:
+    score_dir = config.root / config.score_dir
+    images = lock.index_references()
+    workloads = load_workloads(score_dir, k8s.workloads, WorkloadDocument)
+
+    output_path = config.root / (config.k8s_local_output if local else config.k8s_output)
+    tool_state = output_path.parent / ".score-k8s"
+
+    render_score(
+        config,
+        substrate="k8s",
+        workloads=workloads,
+        rendering_vars={**k8s.vars, STORAGE_CLASS_VAR: LOCAL_STORAGE_CLASS if local else CLOUD_STORAGE_CLASS},
+        templates=[GPU_POD_TEMPLATE, SCALE_TO_ZERO_TEMPLATE],
+        output=output_path,
+        images=images,
+        service_shas=service_shas,
+    )
+
+    # Render the charts through helm template.
+    chart_blocks: list[str] = []
+    with TemporaryDirectory(prefix="resolved-values-") as values_directory:
+        values_dir = Path(values_directory)
+        for chart_name in k8s.charts:
+            # Every values file rides through marker resolution, so the literal-image guard is universal.
+            path = config.root / config.score_dir / "k8s" / "values" / f"{chart_name}.yaml"
+            values_path = values_dir / path.relative_to(config.root)
+            values_path.parent.mkdir(parents=True, exist_ok=True)
+            write_file(
+                values_path,
+                dump_yaml(
+                    resolve_markers(
+                        yaml.safe_load(path.read_text(encoding="utf-8")),
+                        MarkerContext(config.root, images, path.as_posix()),
+                    )
+                ),
+            )
+
+            # Render the chart through helm template with the values file and the locked chart reference.
+            chart_blocks.extend(
+                split_documents(
+                    bash_output(
+                        f"helm template {chart_name} oci://"
+                        f"{lock.require(chart_lock_key(chart_name), f'chart {chart_name!r}')}"
+                        f" --values {values_path.as_posix()}"
+                    )
+                )
+            )
+
+    # Append the rendered charts to the k8s output, preserving any existing content.
+    write_file(
+        output_path,
+        output_path.read_text(encoding="utf-8") + "".join(f"---\n{block}" for block in chart_blocks),
+    )
+
+    # Sort documents by kind, namespace, and name.
+    decorated: list[tuple[str, str, str, int, str]] = []
+    for index, block in enumerate(split_documents(output_path.read_text(encoding="utf-8"))):
+        manifest = ManifestDocument.model_validate(yaml.safe_load(block) or {})
+        decorated.append((manifest.kind, manifest.metadata.namespace, manifest.metadata.name, index, block))
+    decorated.sort()
+    write_file(output_path, "".join(f"---\n{block}" for *_, block in decorated))
+
+    shutil.rmtree(tool_state, ignore_errors=True)
+
+    # Validate the consumed secrets against the declaration.
+    consumed: set[str] = set()
+    for block in split_documents(output_path.read_text(encoding="utf-8")):
+        collect_secret_keys(yaml.safe_load(block), f"{config.project_name}-secrets", consumed)
+    secrets.validate_consumed(consumed, "k8s", SECRETS_FILE.as_posix())
+
+    return output_path
+
+
+def collect_secret_keys(node: Yaml, secret_name: str, consumed: set[str]) -> None:
+    if is_mapping(node):
+        reference = node.get("secretKeyRef")
+        if is_mapping(reference) and reference.get("name") == secret_name:
+            key = reference.get("key")
+            if isinstance(key, str):
+                consumed.add(key)
+
+        for value in node.values():
+            collect_secret_keys(value, secret_name, consumed)
+
+    elif is_list(node):
+        for item in node:
+            collect_secret_keys(item, secret_name, consumed)