fdai-deployment-cli 0.1.2__py3-none-any.whl

fdai_deployment_cli/source_image_stage.py

@@ -0,0 +1,400 @@
+"""Build source-path service images in the deployment registry and bind their digests.
+
+The registry build service builds each baseline service from the pinned source snapshot and
+the registry imports each dependency image by its pinned digest, so the source path needs no
+container engine and writes no kit, signature, SBOM, or provenance statement. Every image is
+read back by digest and by its `sha-<commit>` tag before the receipt binds it. A completed
+receipt permits verification only; an interrupted stage repeats its builds, which only rewrite
+the stage's own commit tags.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+from collections.abc import Callable, Mapping, Sequence
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final, Protocol
+
+from fdai_deployment_cli.contracts import canonical_digest, load_json_object
+from fdai_deployment_cli.private_output import read_private_bytes, write_private_output
+
+SERVICE_ORDER: Final[tuple[str, ...]] = (
+    "core-control-plane",
+    "operator-service",
+    "document-ingestion-api",
+    "document-processing-worker",
+    "isolated-executor",
+)
+DEPENDENCY_IMAGES: Final[Mapping[str, str]] = {
+    "clamav": "docker.io/clamav/clamav@sha256:"
+    "0af8760cd96f9ab67d07977af36e155431581a9fe9f0ec8b256c9f855fda183e",
+    "pgvector": "docker.io/pgvector/pgvector@sha256:"
+    "ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b",
+}
+UNAVAILABLE: Final = "source_image_builder_unavailable"
+FAILED: Final = "source_image_build_failed"
+CHANGED: Final = "source_image_stage_inputs_changed"
+RECEIPT_SCHEMA: Final = "fdai.source-image-stage-receipt.v1"
+CLAIM_SCHEMA: Final = "fdai.source-image-stage-claim.v1"
+
+_COMMIT = re.compile(r"[0-9a-f]{40}")
+_SHA256 = re.compile(r"[0-9a-f]{64}")
+_DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
+_SUBSCRIPTION = re.compile(r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}")
+_REGISTRY = re.compile(r"[a-z0-9]{5,50}")
+_MAX_RECORD_BYTES = 1024 * 1024
+_BUILD_TIMEOUT_SECONDS = 3600
+_IMPORT_TIMEOUT_SECONDS = 1200
+_READ_TIMEOUT_SECONDS = 120
+MAX_PARALLEL_IMAGE_OPERATIONS: Final = 3
+
+_GUIDANCE = {
+    UNAVAILABLE: (
+        "The deployment registry's build service is unavailable in this subscription. Enable "
+        "Azure Container Registry Tasks for the subscription, then rerun the same command."
+    ),
+    FAILED: (
+        "An image build, import, or digest readback failed. Review the registry runs, then "
+        "rerun the same command; the stage rewrites only its own commit tags."
+    ),
+    CHANGED: (
+        "This work directory already holds an image stage for different inputs. Rerun with the "
+        "original source and target, or start a new work directory."
+    ),
+}
+
+
+class SourceImageStageStopped(RuntimeError):
+    """Stop before any service apply with a fixed reason code and guidance."""
+
+    def __init__(self, reason_code: str) -> None:
+        super().__init__(_GUIDANCE[reason_code])
+        self.reason_code = reason_code
+
+
+@dataclass(frozen=True, slots=True)
+class SourceImageTarget:
+    """The deployment's own registry; never a shared or publisher registry."""
+
+    subscription_id: str
+    registry_name: str
+
+    @property
+    def login_server(self) -> str:
+        return f"{self.registry_name}.azurecr.io"
+
+
+@dataclass(frozen=True, slots=True)
+class SourceImageSnapshot:
+    """A verified, pinned source tree for exactly one commit."""
+
+    root: Path
+    commit: str
+    digest: str
+
+
+class RegistryBuildService(Protocol):
+    """The registry operations the stage needs; the stage owns ordering and evidence."""
+
+    def require_available(self, target: SourceImageTarget) -> None: ...
+
+    def build(
+        self, target: SourceImageTarget, *, context: Path, dockerfile: str, image: str
+    ) -> str: ...
+
+    def import_image(self, target: SourceImageTarget, *, source: str, image: str) -> None: ...
+
+    def read_digest(self, target: SourceImageTarget, *, reference: str) -> str: ...
+
+
+def run_source_image_stage(
+    *,
+    snapshot: SourceImageSnapshot,
+    target: SourceImageTarget,
+    builder: RegistryBuildService,
+    work_dir: Path,
+) -> dict[str, object]:
+    """Build, import, and read back every runtime image, or stop before any service apply."""
+
+    _validate(snapshot, target)
+    claim = _claim(snapshot, target)
+    work_dir.mkdir(mode=0o700, exist_ok=True)
+    receipt_path = work_dir / f"source-images-{snapshot.commit}.receipt.json"
+    claim_path = work_dir / f"source-images-{snapshot.commit}.claim.json"
+    if receipt_path.exists():
+        existing = _load(receipt_path)
+        unsigned = {key: value for key, value in existing.items() if key != "receipt_digest"}
+        if existing.get("claim_digest") != claim["claim_digest"] or existing.get(
+            "receipt_digest"
+        ) != canonical_digest(unsigned):
+            raise SourceImageStageStopped(CHANGED)
+        _read_back(builder, target, snapshot.commit, _digests(existing))
+        return existing
+    builder.require_available(target)
+    if claim_path.exists():
+        if _load(claim_path) != claim:
+            raise SourceImageStageStopped(CHANGED)
+    else:
+        write_private_output(claim_path, json.dumps(claim, indent=2, sort_keys=True) + "\n")
+    operations: dict[str, Future[str]] = {}
+    with ThreadPoolExecutor(max_workers=MAX_PARALLEL_IMAGE_OPERATIONS) as executor:
+        for service in SERVICE_ORDER:
+            operations[service] = executor.submit(
+                _build_service,
+                builder,
+                target,
+                snapshot,
+                service,
+            )
+        for name, source in DEPENDENCY_IMAGES.items():
+            operations[name] = executor.submit(
+                _import_dependency,
+                builder,
+                target,
+                snapshot.commit,
+                name,
+                source,
+            )
+        digests = {name: _require_digest(future.result()) for name, future in operations.items()}
+    _read_back(builder, target, snapshot.commit, digests)
+    receipt: dict[str, object] = {
+        "schema_version": RECEIPT_SCHEMA,
+        "claim_digest": claim["claim_digest"],
+        "source_commit": snapshot.commit,
+        "snapshot_digest": snapshot.digest,
+        "registry_login_server": target.login_server,
+        "image_digests": dict(sorted(digests.items())),
+        "image_refs": {
+            name: f"{target.login_server}/{name}@{digest}"
+            for name, digest in sorted(digests.items())
+        },
+        "builder": "azure-container-registry-tasks",
+        "max_parallel_image_operations": MAX_PARALLEL_IMAGE_OPERATIONS,
+        "provenance": "operator-selected-source",
+        "release_signature_verified": False,
+        "effect_verified": True,
+        "mutation_performed": True,
+        "deployment_ready": False,
+    }
+    receipt["receipt_digest"] = canonical_digest(receipt)
+    write_private_output(receipt_path, json.dumps(receipt, indent=2, sort_keys=True) + "\n")
+    return receipt
+
+
+def _build_service(
+    builder: RegistryBuildService,
+    target: SourceImageTarget,
+    snapshot: SourceImageSnapshot,
+    service: str,
+) -> str:
+    return builder.build(
+        target,
+        context=snapshot.root,
+        dockerfile=f"services/{service}/docker/Dockerfile",
+        image=f"{service}:sha-{snapshot.commit}",
+    )
+
+
+def _import_dependency(
+    builder: RegistryBuildService,
+    target: SourceImageTarget,
+    commit: str,
+    name: str,
+    source: str,
+) -> str:
+    builder.import_image(target, source=source, image=f"{name}:sha-{commit}")
+    return source.rsplit("@", 1)[1]
+
+
+def _validate(snapshot: SourceImageSnapshot, target: SourceImageTarget) -> None:
+    if (
+        _COMMIT.fullmatch(snapshot.commit) is None
+        or _SHA256.fullmatch(snapshot.digest) is None
+        or _SUBSCRIPTION.fullmatch(target.subscription_id) is None
+        or _REGISTRY.fullmatch(target.registry_name) is None
+    ):
+        raise ValueError("source image stage inputs are malformed")
+    for service in SERVICE_ORDER:
+        dockerfile = snapshot.root / "services" / service / "docker" / "Dockerfile"
+        if dockerfile.is_symlink() or not dockerfile.is_file():
+            raise ValueError(f"source snapshot lacks the {service} Dockerfile")
+
+
+def _claim(snapshot: SourceImageSnapshot, target: SourceImageTarget) -> dict[str, object]:
+    claim: dict[str, object] = {
+        "schema_version": CLAIM_SCHEMA,
+        "source_commit": snapshot.commit,
+        "snapshot_digest": snapshot.digest,
+        "subscription_id": target.subscription_id,
+        "registry_login_server": target.login_server,
+        "services": list(SERVICE_ORDER),
+        "dependency_images": dict(sorted(DEPENDENCY_IMAGES.items())),
+    }
+    claim["claim_digest"] = canonical_digest(claim)
+    return claim
+
+
+def _load(path: Path) -> dict[str, object]:
+    data = read_private_bytes(path, max_bytes=_MAX_RECORD_BYTES)
+    return dict(load_json_object(data, label=path.name, max_bytes=_MAX_RECORD_BYTES))
+
+
+def _digests(receipt: Mapping[str, object]) -> dict[str, str]:
+    raw = receipt.get("image_digests")
+    expected = set(SERVICE_ORDER) | set(DEPENDENCY_IMAGES)
+    if not isinstance(raw, dict) or set(raw) != expected:
+        raise SourceImageStageStopped(CHANGED)
+    return {str(name): _require_digest(value) for name, value in raw.items()}
+
+
+def _require_digest(value: object) -> str:
+    if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+        raise SourceImageStageStopped(FAILED)
+    return value
+
+
+def _read_back(
+    builder: RegistryBuildService,
+    target: SourceImageTarget,
+    commit: str,
+    digests: Mapping[str, str],
+) -> None:
+    for name, digest in sorted(digests.items()):
+        by_digest = builder.read_digest(target, reference=f"{name}@{digest}")
+        by_tag = builder.read_digest(target, reference=f"{name}:sha-{commit}")
+        if by_digest != digest or by_tag != digest:
+            raise SourceImageStageStopped(FAILED)
+
+
+Runner = Callable[..., "subprocess.CompletedProcess[str]"]
+
+
+def _run(
+    command: Sequence[str], timeout: int, *, cwd: Path | None = None
+) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        list(command), check=False, capture_output=True, text=True, timeout=timeout, cwd=cwd
+    )
+
+
+class AzureRegistryBuildService:
+    """Azure Container Registry Tasks and import through the signed-in Azure CLI.
+
+    Every call carries the explicit subscription. Failures surface as fixed reason codes;
+    command output never reaches the error text.
+    """
+
+    def __init__(self, runner: Runner = _run) -> None:
+        self._runner = runner
+
+    def require_available(self, target: SourceImageTarget) -> None:
+        result = self._az(
+            target,
+            ("acr", "show", "--name", target.registry_name),
+            ("--query", "{loginServer:loginServer,state:provisioningState}", "--output", "json"),
+            timeout=_READ_TIMEOUT_SECONDS,
+            reason=UNAVAILABLE,
+        )
+        try:
+            registry = json.loads(result)
+        except json.JSONDecodeError:
+            raise SourceImageStageStopped(UNAVAILABLE) from None
+        if registry != {"loginServer": target.login_server, "state": "Succeeded"}:
+            raise SourceImageStageStopped(UNAVAILABLE)
+
+    def build(
+        self, target: SourceImageTarget, *, context: Path, dockerfile: str, image: str
+    ) -> str:
+        output = self._az(
+            target,
+            ("acr", "build", "--registry", target.registry_name, "--image", image),
+            (
+                "--file",
+                dockerfile,
+                "--platform",
+                "linux/amd64",
+                "--timeout",
+                str(_BUILD_TIMEOUT_SECONDS),
+                "--no-logs",
+                "--output",
+                "json",
+                str(context),
+            ),
+            timeout=_BUILD_TIMEOUT_SECONDS + 300,
+            reason=FAILED,
+            # The Azure CLI resolves a relative --file against the working directory.
+            cwd=context,
+        )
+        try:
+            run = json.loads(output)
+        except json.JSONDecodeError:
+            raise SourceImageStageStopped(FAILED) from None
+        repository, tag = image.split(":", 1)
+        if not isinstance(run, dict) or run.get("status") != "Succeeded":
+            raise SourceImageStageStopped(FAILED)
+        matches = [
+            record.get("digest")
+            for record in run.get("outputImages") or []
+            if isinstance(record, dict)
+            and record.get("repository") == repository
+            and record.get("tag") == tag
+            and record.get("registry") == target.login_server
+        ]
+        if len(matches) != 1:
+            raise SourceImageStageStopped(FAILED)
+        return _require_digest(matches[0])
+
+    def import_image(self, target: SourceImageTarget, *, source: str, image: str) -> None:
+        self._az(
+            target,
+            ("acr", "import", "--name", target.registry_name, "--source", source),
+            ("--image", image, "--force"),
+            timeout=_IMPORT_TIMEOUT_SECONDS,
+            reason=FAILED,
+        )
+
+    def read_digest(self, target: SourceImageTarget, *, reference: str) -> str:
+        output = self._az(
+            target,
+            ("acr", "manifest", "show-metadata", "--registry", target.registry_name),
+            ("--name", reference, "--query", "digest", "--output", "tsv"),
+            timeout=_READ_TIMEOUT_SECONDS,
+            reason=FAILED,
+        )
+        return _require_digest(output.strip())
+
+    def _az(
+        self,
+        target: SourceImageTarget,
+        head: tuple[str, ...],
+        tail: tuple[str, ...],
+        *,
+        timeout: int,
+        reason: str,
+        cwd: Path | None = None,
+    ) -> str:
+        command = (
+            "az",
+            *head,
+            "--subscription",
+            target.subscription_id,
+            *tail,
+            "--only-show-errors",
+        )
+        try:
+            result = (
+                self._runner(command, timeout)
+                if cwd is None
+                else self._runner(command, timeout, cwd=cwd)
+            )
+        except subprocess.TimeoutExpired:
+            raise SourceImageStageStopped(reason) from None
+        if result.returncode != 0:
+            if "TasksOperationsNotAllowed" in (result.stderr or ""):
+                raise SourceImageStageStopped(UNAVAILABLE)
+            raise SourceImageStageStopped(reason)
+        return result.stdout

fdai_deployment_cli/source_input.py

@@ -0,0 +1,173 @@
+"""Pin operator-selected Git inputs without claiming release signature trust."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import stat
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+from fdai_deployment_cli.contracts import canonical_digest
+
+_COMMIT = re.compile(r"[0-9a-f]{40}")
+_MAX_FILES = 65_536
+_MAX_FILE_BYTES = 64 * 1024 * 1024
+_MAX_SOURCE_BYTES = 2 * 1024 * 1024 * 1024
+
+
+@dataclass(frozen=True, slots=True)
+class SourceDeploymentInput:
+    """Exact source identity; a local content pin, not publisher trust or approval."""
+
+    root: Path
+    commit: str
+    tree: str
+    content_digest: str
+    file_count: int
+
+    @property
+    def digest(self) -> str:
+        """Bind portable source evidence without exposing the local checkout path."""
+        return canonical_digest(self.to_mapping())
+
+    def to_mapping(self) -> dict[str, object]:
+        """Return secret-free provenance for reviews and retained source runs."""
+        return {
+            "schema_version": "fdai.source-deployment-input.v1",
+            "provenance": "operator-selected-source",
+            "source_commit": self.commit,
+            "source_tree": self.tree,
+            "content_digest": self.content_digest,
+            "file_count": self.file_count,
+            "release_signature_verified": False,
+        }
+
+    def reverify(self) -> None:
+        """Reject changed source before another build, transfer, or deployment effect."""
+        if inspect_source(self.root, expected_commit=self.commit) != self:
+            raise ValueError("source deployment inputs changed; preserve the existing run")
+
+
+def inspect_source(root: Path, *, expected_commit: str | None = None) -> SourceDeploymentInput:
+    """Verify a complete clean checkout against raw Git blobs, without executing its code.
+
+    Git clean filters and index flags cannot hide raw content changes. External links,
+    submodules, hardlinks, untracked files and oversized inputs are rejected. Ignored
+    files are never part of the evidence and must not enter a later build snapshot.
+    """
+    if root.is_symlink():
+        raise ValueError("source deployment requires a non-symlink checkout root")
+    root = root.resolve(strict=True)
+    if _git(root, "rev-parse", "--show-toplevel").decode().strip() != str(root):
+        raise ValueError("source deployment must select the checkout root")
+    commit = _git(root, "rev-parse", "--verify", "HEAD^{commit}").decode().strip()
+    if _COMMIT.fullmatch(commit) is None or expected_commit not in {None, commit}:
+        raise ValueError("source deployment revision differs from the selected commit")
+    if _git(root, "status", "--porcelain=v1", "--untracked-files=all"):
+        raise ValueError("source deployment requires a clean committed checkout")
+    tree = _git(root, "rev-parse", "HEAD^{tree}").decode().strip()
+    entries = _git(root, "ls-tree", "-rz", "--full-tree", commit).split(b"\0")[:-1]
+    if not 0 < len(entries) <= _MAX_FILES:
+        raise ValueError("source deployment file inventory is empty or exceeds its bound")
+    tracked = {entry.split(b"\t", 1)[1].decode("utf-8") for entry in entries}
+    records: list[dict[str, str]] = []
+    total = 0
+    for entry in entries:
+        metadata, raw_path = entry.split(b"\t", 1)
+        mode, kind, blob_digest = metadata.decode("ascii").split()
+        path = raw_path.decode("utf-8")
+        if kind != "blob" or mode not in {"100644", "100755", "120000"}:
+            raise ValueError("source deployment does not accept external submodules")
+        content = _read_tracked(root, path, mode=mode)
+        if mode == "120000":
+            try:
+                resolved = (root / path).resolve(strict=True).relative_to(root).as_posix()
+            except (OSError, ValueError, RuntimeError):
+                raise ValueError("source link must resolve inside the checkout") from None
+            if resolved not in tracked:
+                raise ValueError("source link target must be a tracked file")
+        total += len(content)
+        if total > _MAX_SOURCE_BYTES:
+            raise ValueError("source deployment exceeds its total size bound")
+        blob = b"blob " + str(len(content)).encode("ascii") + b"\0" + content
+        if hashlib.sha1(blob, usedforsecurity=False).hexdigest() != blob_digest:
+            raise ValueError("source bytes differ from the committed input")
+        records.append({"path": path, "mode": mode, "sha256": hashlib.sha256(content).hexdigest()})
+    if _git(root, "rev-parse", "HEAD").decode().strip() != commit:
+        raise ValueError("source revision changed during inspection")
+    return SourceDeploymentInput(
+        root, commit, tree, canonical_digest({"files": records}), len(records)
+    )
+
+
+def _read_tracked(root: Path, relative: str, *, mode: str) -> bytes:
+    parts = PurePosixPath(relative).parts
+    if not parts or relative.startswith("/") or any(part in {".", ".."} for part in parts):
+        raise ValueError("source input path is invalid")
+    parent = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+    try:
+        for part in parts[:-1]:
+            child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent)
+            os.close(parent)
+            parent = child
+        if mode == "120000":
+            before_link = os.stat(parts[-1], dir_fd=parent, follow_symlinks=False)
+            target = os.readlink(parts[-1], dir_fd=parent)
+            after_link = os.stat(parts[-1], dir_fd=parent, follow_symlinks=False)
+            stable_fields = ("st_dev", "st_ino", "st_mode", "st_size", "st_mtime_ns", "st_ctime_ns")
+            if (
+                any(
+                    getattr(before_link, field) != getattr(after_link, field)
+                    for field in stable_fields
+                )
+                or Path(target).is_absolute()
+            ):
+                raise ValueError("source link changed or has an absolute target")
+            return os.fsencode(target)
+        descriptor = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent)
+        with os.fdopen(descriptor, "rb") as stream:
+            before = os.fstat(stream.fileno())
+            if (
+                not stat.S_ISREG(before.st_mode)
+                or before.st_nlink != 1
+                or before.st_size > _MAX_FILE_BYTES
+                or bool(before.st_mode & stat.S_IXUSR) != (mode == "100755")
+            ):
+                raise ValueError("source input type, mode, or size is invalid")
+            content = stream.read(_MAX_FILE_BYTES + 1)
+            after = os.fstat(stream.fileno())
+            if (
+                len(content) != before.st_size
+                or after.st_size != before.st_size
+                or after.st_mtime_ns != before.st_mtime_ns
+                or after.st_ctime_ns != before.st_ctime_ns
+            ):
+                raise ValueError("source input changed during inspection")
+            return content
+    finally:
+        os.close(parent)
+
+
+def _git(root: Path, *arguments: str) -> bytes:
+    environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
+    environment["GIT_TERMINAL_PROMPT"] = "0"
+    try:
+        result = subprocess.run(
+            ["git", *arguments],
+            cwd=root,
+            env=environment,
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            check=True,
+            timeout=30,
+        )
+    except (OSError, subprocess.SubprocessError):
+        raise ValueError(
+            "source checkout could not be verified; no deployment was started"
+        ) from None
+    if len(result.stdout) > 16 * 1024 * 1024:
+        raise ValueError("source Git response exceeds its size bound")
+    return result.stdout