dirigent-block-execute 0.17.1__py3-none-any.whl

dirigent_block_execute/__init__.py

@@ -0,0 +1,43 @@
+"""The execute block family: what runs code on the worker, in a shell, a container, or a checkout."""
+
+from dirigent_block_execute.build import DockerBuildOperator
+from dirigent_block_execute.compose import DockerComposeDownOperator, DockerComposeUpOperator
+from dirigent_block_execute.docker import DockerConnectionKind, DockerRunOperator
+from dirigent_block_execute.git import GitCheckoutOperator, GitConnectionKind
+from dirigent_block_execute.shell import ShellRunOperator
+from dirigent_plugin import Contribution, extension
+
+
+class ExecuteBlocks:
+    """The plugin object the host discovers under the dirigent.plugins.v1 entry-point group."""
+
+    @extension
+    def contribute(self) -> Contribution:
+        """Contribute the blocks that run something on the worker, and their connection kinds."""
+        return Contribution(
+            operators=[
+                ShellRunOperator(),
+                DockerRunOperator(),
+                DockerComposeUpOperator(),
+                DockerComposeDownOperator(),
+                DockerBuildOperator(),
+                GitCheckoutOperator(),
+            ],
+            connection_kinds=[DockerConnectionKind(), GitConnectionKind()],
+        )
+
+
+plugin = ExecuteBlocks()
+
+__all__ = [
+    "DockerBuildOperator",
+    "DockerComposeDownOperator",
+    "DockerComposeUpOperator",
+    "DockerConnectionKind",
+    "DockerRunOperator",
+    "ExecuteBlocks",
+    "GitCheckoutOperator",
+    "GitConnectionKind",
+    "ShellRunOperator",
+    "plugin",
+]

dirigent_block_execute/build.py

@@ -0,0 +1,319 @@
+"""``docker.build``: build an image from a context on the worker, with buildx.
+
+The block shells out to ``docker buildx build`` -- BuildKit, its cache and multi-stage builds
+come for free -- and reads the built image's id from an ``--iidfile`` rather than scraping the
+log. The image lands in the worker's own daemon store, so a later ``docker.run`` or
+``docker.compose`` step on the same worker references it by tag.
+
+Reaching the Docker daemon is reaching root on the host when the daemon is the host's own, so
+the block declares ``local_execution`` and the engine refuses it unless the instance allowlists
+its id.
+
+``push`` needs a ``docker`` connection carrying a registry credential, and is refused without
+one. The login goes to a ``DOCKER_CONFIG`` directory of its own under the run's work directory --
+never the worker's own config -- with the password on stdin rather than in an argument, and the
+directory and the session in it go when the step leaves.
+"""
+
+import re
+from datetime import timedelta
+from pathlib import Path
+from typing import ClassVar
+
+from pydantic import BaseModel, Field, model_validator
+
+from dirigent_block_execute import subprocess
+from dirigent_block_execute.capture import log_stream, scrub, tail
+from dirigent_block_execute.docker import (
+    DAEMON_ENV,
+    DockerConnectionConfig,
+    Sealed,
+    daemon_environment,
+    login,
+    logout,
+    sealed,
+    write_cli_config,
+)
+from dirigent_block_execute.environment import reject_reserved
+from dirigent_block_execute.messages import (
+    BUILD_EXITED,
+    BUILD_PATHS_STAY_INSIDE,
+    LOGIN_FAILED,
+    NO_IMAGE_ID,
+    NO_REGISTRY_CREDENTIAL,
+    PUSH_EXITED,
+    PUSH_NEEDS_A_CONNECTION,
+    PUSH_NEEDS_A_TAG,
+)
+from dirigent_common import BlockModel, Duration
+from dirigent_plugin import (
+    BlockFailure,
+    ConnectionRef,
+    ErrorClass,
+    Operator,
+    OperatorSpec,
+    RemoteHandle,
+    StepContext,
+)
+
+#: What a daemon that is not there or not ours says, a transient condition rather than a broken build.
+DAEMON_UNREACHABLE = ("cannot connect to the docker daemon", "is the docker daemon running", "permission denied")
+
+#: What ``docker push`` prints once the registry has taken a tag, which is where its digest is.
+PUSHED_DIGEST = re.compile(r"digest:\s*(sha256:[0-9a-f]{64})")
+
+
+class DockerBuildConfig(BlockModel):
+    """Which context to build, with what Dockerfile, tags, and build arguments."""
+
+    context: str
+    """The build context, as a directory inside the run's work directory; never absolute, never climbing out."""
+
+    dockerfile: str = "Dockerfile"
+    """The Dockerfile, as a path relative to the context."""
+
+    tags: list[str] = Field(default_factory=list[str])
+    """Tags to give the built image (``--tag``); a later step references the image by one of them."""
+
+    build_args: dict[str, str] = Field(default_factory=dict[str, str])
+    """Build arguments the Dockerfile reads (``--build-arg``)."""
+
+    target: str | None = None
+    """The stage to stop at in a multi-stage build (``--target``)."""
+
+    platform: str | None = None
+    """The platform to build for, such as ``linux/amd64`` (``--platform``)."""
+
+    pull: bool = False
+    """Always attempt to pull a newer version of the base image (``--pull``)."""
+
+    no_cache: bool = False
+    """Build every layer from scratch, ignoring the cache (``--no-cache``)."""
+
+    push: bool = False
+    """Push every tag the build produced to the registry the connection names.
+
+    Needs a ``connection`` carrying a registry username and password: an anonymous push is
+    refused rather than attempted."""
+
+    connection: ConnectionRef | None = None
+    """A ``docker`` connection naming the daemon to build on, the registry to push to, or both.
+
+    Absent, the build runs against whatever daemon the worker's own environment names and
+    nothing is pushed."""
+
+    env: dict[str, str] = Field(default_factory=dict[str, str])
+    """Variables set for the CLI itself, such as BuildKit's own toggles."""
+
+    env_allowlist: list[str] = Field(default_factory=list[str])
+    """Worker environment variables the CLI is allowed to inherit; never the instance's ``DIRIGENT_*``."""
+
+    command_path: list[str] = Field(default_factory=lambda: ["docker", "buildx", "build"])
+    """The CLI to invoke, for a host that wraps buildx."""
+
+    timeout: Duration = timedelta(minutes=30)
+    """The overall deadline on the build, after which it is killed as transient."""
+
+    @model_validator(mode="after")
+    def _check_shape(self) -> "DockerBuildConfig":
+        """Refuse an unbacked push, a context or Dockerfile that climbs out, and a reserved env inheritance."""
+        if self.push and self.connection is None:
+            raise ValueError(PUSH_NEEDS_A_CONNECTION.render())
+        if self.push and not self.tags:
+            raise ValueError(PUSH_NEEDS_A_TAG.render())
+        for path in (self.context, self.dockerfile):
+            if Path(path).is_absolute() or ".." in Path(path).parts:
+                raise ValueError(BUILD_PATHS_STAY_INSIDE.render())
+        reject_reserved(self.env_allowlist)
+        return self
+
+
+class DockerBuildOutput(BlockModel):
+    """The image the build produced, with its full log addressable as an artifact."""
+
+    image_id: str
+    """The built image's id, read from the ``--iidfile`` the daemon wrote, not scraped from the log."""
+
+    tags: list[str] = Field(default_factory=list[str])
+    """The tags the image was given, by which a later step on the same worker references it."""
+
+    size_bytes: int = 0
+    """The image's size on disk, from the daemon; zero when it could not be read."""
+
+    pushed: list[str] = Field(default_factory=list[str])
+    """The tags that reached the registry, empty when the step pushed nothing."""
+
+    digests: dict[str, str] = Field(default_factory=dict[str, str])
+    """The registry's digest for each pushed tag, for the tags the CLI reported one for."""
+
+    build_log_uri: str
+    """Where the whole of the build log (buildx's progress on stderr) was written."""
+
+    stdout_uri: str
+    """Where the whole of the CLI's stdout was written."""
+
+
+class DockerBuildOperator(Operator[DockerBuildConfig, DockerBuildOutput]):
+    """Builds an image from a context in the run's work directory with buildx, behind the allowlist."""
+
+    spec = OperatorSpec(
+        id="docker.build",
+        group="execute",
+        summary="Build a container image on the worker.",
+        idempotent=False,
+        local_execution=True,
+    )
+    config_model: ClassVar[type[BaseModel]] = DockerBuildConfig
+    output_model: ClassVar[type[BaseModel]] = DockerBuildOutput
+
+    async def execute(self, config: DockerBuildConfig, ctx: StepContext) -> DockerBuildOutput | RemoteHandle:
+        """Build the image, read its id from the iidfile, and report its size and tags."""
+        root = ctx.work
+        context = root / config.context
+        iidfile = Path(f"{root / subprocess.segment(ctx, 'build')}.iid")
+        iidfile.parent.mkdir(parents=True, exist_ok=True)
+        settings = ctx.connection(config.connection, DockerConnectionConfig) if config.connection else None
+        if config.push and (settings is None or not settings.authenticates):
+            raise BlockFailure(NO_REGISTRY_CREDENTIAL, error_class=ErrorClass.REJECTED)
+        timeout = config.timeout.total_seconds()
+        artifacts = subprocess.prefix(ctx, "build")
+        stdout_uri = f"{artifacts}-stdout.txt"
+        build_log_uri = f"{artifacts}-build.log"
+
+        with sealed(settings, root, isolate_config=config.push) as material:
+            environ = daemon_environment(
+                subprocess.environment([*DAEMON_ENV, *config.env_allowlist], config.env, root), material
+            )
+            write_cli_config(root / ".docker")
+            code, out, err = await subprocess.run(
+                directory=root,
+                ctx=ctx,
+                stdout_uri=stdout_uri,
+                stderr_uri=build_log_uri,
+                timeout_seconds=timeout,
+                environ=environ,
+                argv=build_argv(config, context, iidfile),
+                what="docker buildx build",
+                redact=material.secrets,
+            )
+            log_stream(ctx, "stdout", out, material.secrets)
+            log_stream(ctx, "stderr", err, material.secrets)
+            ctx.log.info("docker build finished", exit_code=code, tags=", ".join(config.tags))
+            if code != 0:
+                detail = (
+                    tail(err.tail, redact=material.secrets) or tail(out.tail, redact=material.secrets) or "no output"
+                )
+                raise BlockFailure(BUILD_EXITED, error_class=_classify(err.tail), code=code, detail=detail)
+
+            image_id = iidfile.read_text().strip() if iidfile.exists() else ""
+            if not image_id:
+                raise BlockFailure(NO_IMAGE_ID, error_class=ErrorClass.UNKNOWN)
+            size_bytes = await _image_size(config, image_id, root, environ, timeout)
+            pushed, digests = await _push(config, settings, root, environ, material, timeout, ctx)
+        ctx.log.info("image built", image_id=image_id[:19], size_bytes=size_bytes, tags=", ".join(config.tags))
+        return DockerBuildOutput(
+            image_id=image_id,
+            tags=config.tags,
+            size_bytes=size_bytes,
+            pushed=pushed,
+            digests=digests,
+            build_log_uri=build_log_uri,
+            stdout_uri=stdout_uri,
+        )
+
+
+def build_argv(config: DockerBuildConfig, context: Path, iidfile: Path) -> list[str]:
+    """Assemble the ``buildx build`` argv, flag by flag, with the context last."""
+    argv = [*config.command_path, "--file", str(context / config.dockerfile), "--iidfile", str(iidfile)]
+    for tag in config.tags:
+        argv += ["--tag", tag]
+    for name, value in config.build_args.items():
+        argv += ["--build-arg", f"{name}={value}"]
+    if config.target is not None:
+        argv += ["--target", config.target]
+    if config.platform is not None:
+        argv += ["--platform", config.platform]
+    if config.pull:
+        argv.append("--pull")
+    if config.no_cache:
+        argv.append("--no-cache")
+    # Load the result into the worker's daemon store, so a later step can run it by tag; the
+    # container-driver default would otherwise leave the image only in the builder's cache.
+    argv.append("--load")
+    argv.append(str(context))
+    return argv
+
+
+async def _push(
+    config: DockerBuildConfig,
+    settings: DockerConnectionConfig | None,
+    root: Path,
+    environ: dict[str, str],
+    material: Sealed,
+    timeout: float,
+    ctx: StepContext,
+) -> tuple[list[str], dict[str, str]]:
+    """Log in, push every tag the build produced, and log out again.
+
+    The login lives in the ``DOCKER_CONFIG`` directory the sealed material made, so it reaches
+    neither the worker's own config nor any later step, and the logout takes even that away.
+    """
+    if not config.push:
+        return [], {}
+    assert settings is not None
+    docker = config.command_path[0] if config.command_path else "docker"
+    code, err = await login(docker, root, environ, settings)
+    if code != 0:
+        detail = tail(err, redact=material.secrets) or f"docker login exited {code}"
+        raise BlockFailure(
+            LOGIN_FAILED, error_class=ErrorClass.REJECTED, registry=settings.registry_name, detail=detail
+        )
+    pushed: list[str] = []
+    digests: dict[str, str] = {}
+    try:
+        for tag in config.tags:
+            code, out, err = await subprocess.output(
+                argv=[docker, "push", tag],
+                directory=root,
+                environ=environ,
+                timeout_seconds=timeout,
+                what="docker push",
+            )
+            if code != 0:
+                detail = tail(err, redact=material.secrets) or tail(out, redact=material.secrets) or "no output"
+                raise BlockFailure(PUSH_EXITED, error_class=_classify(err), tag=tag, code=code, detail=detail)
+            found = PUSHED_DIGEST.search(scrub(out.decode("utf-8", errors="replace"), material.secrets))
+            if found is not None:
+                digests[tag] = found.group(1)
+            pushed.append(tag)
+            ctx.log.info("image pushed", tag=tag, registry=settings.registry_name, digest=digests.get(tag, ""))
+    finally:
+        await logout(docker, root, environ, settings)
+    return pushed, digests
+
+
+async def _image_size(
+    config: DockerBuildConfig, image_id: str, root: Path, environ: dict[str, str], timeout: float
+) -> int:
+    """Read the built image's size from the daemon, best effort; zero when it cannot be read."""
+    docker = config.command_path[0] if config.command_path else "docker"
+    try:
+        _code, out, _err = await subprocess.output(
+            argv=[docker, "image", "inspect", "--format", "{{.Size}}", image_id],
+            directory=root,
+            environ=environ,
+            timeout_seconds=timeout,
+            what="docker image inspect",
+        )
+    except BlockFailure:
+        return 0
+    text = out.decode("utf-8", errors="replace").strip()
+    return int(text) if text.isdigit() else 0
+
+
+def _classify(stderr_tail: bytes) -> ErrorClass:
+    """A daemon that cannot be reached is transient; a build that would not compile is not."""
+    text = stderr_tail.decode("utf-8", errors="replace").lower()
+    if any(marker in text for marker in DAEMON_UNREACHABLE):
+        return ErrorClass.TRANSIENT
+    return ErrorClass.UNKNOWN

dirigent_block_execute/capture.py

@@ -0,0 +1,289 @@
+"""Capturing what a local process printed: decode it, log it, store it, quote it."""
+
+import asyncio
+from collections.abc import Sequence
+from typing import NamedTuple
+
+from dirigent_plugin import ByteSink, StepContext
+
+LOGGED_LINES = 40
+
+FAILURE_TAIL_LINES = 5
+
+#: What a scrubbed secret is replaced by, matching the marker the API redacts a connection with.
+REDACTED = "***"
+
+
+class Captured(NamedTuple):
+    """One stream as an output carries it: the head that inlines, and the truth about the rest."""
+
+    text: str
+    """The head of the stream, as much of it as the instance lets a block inline."""
+
+    total_bytes: int
+    """How much the process printed, whether it inlined or not."""
+
+    truncated: bool
+    """Whether the text above is short of the stream; the artifact URI always holds all of it."""
+
+
+def captured(payload: bytes, limit: int) -> Captured:
+    """Cut one captured stream down to what may be inlined, and say what was cut."""
+    return Captured(text=decode(payload[:limit]), total_bytes=len(payload), truncated=len(payload) > limit)
+
+
+#: How much of the end of a stream is kept, for the lines a failure message quotes. The head
+#: is what a block may inline; this is the other end, and together they bound what is held.
+TAIL_BYTES = 8 * 1024
+
+#: How much is read from a pipe at a time.
+CHUNK_BYTES = 64 * 1024
+
+
+class Logged(NamedTuple):
+    """What a stream put into the run log while it was still being read."""
+
+    lines: int
+    """Non-blank lines the stream carried."""
+
+    live: int
+    """How many of them reached the run log as they arrived."""
+
+
+class Drained(NamedTuple):
+    """What was kept of a stream that was written through to storage as it arrived."""
+
+    head: bytes
+    """The beginning, as much as may be inlined."""
+
+    tail: bytes
+    """The end, for the lines a failure quotes."""
+
+    total_bytes: int
+    """How much the process printed, all of which reached storage."""
+
+    logged: Logged | None = None
+    """What was already logged live, or ``None`` when the stream was not logged as it arrived."""
+
+    def captured(self, limit: int) -> Captured:
+        """Present the head as an output carries it, against what the whole stream was."""
+        return Captured(
+            text=decode(self.head[:limit]),
+            total_bytes=self.total_bytes,
+            truncated=self.total_bytes > limit,
+        )
+
+
+class Ends:
+    """Both ends of a stream and its size, holding nothing of the middle.
+
+    A stream is fed in as it arrives. What is kept is the head, which is what may be inlined,
+    and the last few kilobytes, which is what a failure message quotes.
+    """
+
+    def __init__(self, head_limit: int) -> None:
+        """Keep at most this much of the beginning."""
+        self._head_limit = head_limit
+        self._head = bytearray()
+        self._tail = bytearray()
+        self.total_bytes = 0
+
+    def add(self, chunk: bytes) -> None:
+        """Take one piece of the stream."""
+        self.total_bytes += len(chunk)
+        if len(self._head) < self._head_limit:
+            self._head.extend(chunk[: self._head_limit - len(self._head)])
+        self._tail.extend(chunk)
+        if len(self._tail) > TAIL_BYTES:
+            del self._tail[: len(self._tail) - TAIL_BYTES]
+
+    def drained(self, live: "LiveLines | None" = None) -> Drained:
+        """Present what was kept, and what a live logger already said about it."""
+        return Drained(
+            head=bytes(self._head),
+            tail=bytes(self._tail),
+            total_bytes=self.total_bytes,
+            logged=live.logged() if live is not None else None,
+        )
+
+
+class LiveLines:
+    """Puts the first lines of a stream into the run log as the process prints them.
+
+    Bytes are fed in as they are read. Every complete non-blank line is counted, and the
+    first ``budget`` of them are logged where they happen, so a run screen shows a long
+    command working rather than everything it printed at the moment it exited. Past the
+    budget nothing is held: the line is counted and dropped, and the other end of the stream
+    reaches the log when the stream closes.
+    """
+
+    def __init__(
+        self,
+        ctx: StepContext,
+        stream: str,
+        *,
+        budget: int = LOGGED_LINES // 2,
+        redact: Sequence[str] = (),
+    ) -> None:
+        """Log to this stream's level, redacting the strings the block handed the process."""
+        self._log = ctx.log.info if stream == "stdout" else ctx.log.warning
+        self._stream = stream
+        self._budget = budget
+        self._redact = redact
+        self._pending = bytearray()
+        self._nonblank = False
+        self._lines = 0
+        self._live = 0
+
+    def feed(self, chunk: bytes) -> None:
+        """Take one piece of the stream, logging every complete line it completes."""
+        start = 0
+        while (cut := chunk.find(b"\n", start)) >= 0:
+            self._take(chunk[start:cut])
+            self._finish()
+            start = cut + 1
+        self._take(chunk[start:])
+
+    def close(self) -> None:
+        """Log a last line the process left without a newline."""
+        self._finish()
+
+    def logged(self) -> Logged:
+        """How many lines the stream carried and how many of them are in the log already."""
+        return Logged(lines=self._lines, live=self._live)
+
+    def _take(self, piece: bytes) -> None:
+        """Hold a piece of the line being read, or only whether it has anything in it."""
+        if not piece:
+            return
+        self._nonblank = self._nonblank or bool(piece.strip())
+        if self._live < self._budget:
+            self._pending.extend(piece)
+
+    def _finish(self) -> None:
+        """End the line being read: count it, and log it while the budget lasts."""
+        if self._nonblank:
+            self._lines += 1
+            if self._live < self._budget:
+                self._live += 1
+                self._log(scrub(decode(bytes(self._pending)).rstrip("\r"), self._redact), stream=self._stream)
+        self._pending.clear()
+        self._nonblank = False
+
+
+async def drain(
+    reader: "asyncio.StreamReader | None",
+    sink: ByteSink,
+    *,
+    head_limit: int,
+    live: LiveLines | None = None,
+) -> Drained:
+    """Read one pipe to its end, writing it through to storage and keeping both ends.
+
+    Nothing holds the whole stream. A command that prints a gigabyte is a file in storage and
+    two bounded buffers here, rather than a gigabyte in the worker.
+
+    With a ``live`` logger the lines go into the run log as they are read, up to its budget.
+    """
+    ends = Ends(head_limit)
+    while reader is not None:
+        chunk = await reader.read(CHUNK_BYTES)
+        if not chunk:
+            break
+        ends.add(chunk)
+        if live is not None:
+            live.feed(chunk)
+        await sink.write(chunk)
+    if live is not None:
+        live.close()
+    return ends.drained(live)
+
+
+def decode(payload: bytes) -> str:
+    """Render captured bytes as text without ever failing on what a process printed."""
+    return payload.decode("utf-8", errors="replace")
+
+
+def lines_of(payload: bytes) -> list[str]:
+    """Split a captured stream into its non-blank lines."""
+    return [line for line in decode(payload).splitlines() if line.strip()]
+
+
+def scrub(text: str, secrets: Sequence[str] = ()) -> str:
+    """Replace every occurrence of a secret with the redaction marker.
+
+    A tool prints what it was given. ``git`` echoes a remote URL, an ssh command line and,
+    when a credential is wrong, sometimes the credential itself, so a block that hands a
+    process a secret hands this the same strings and nothing reaches a log or a failure
+    message with them still in it. Longest first, so a secret that contains another is
+    replaced whole rather than left in pieces.
+    """
+    for secret in sorted((one for one in secrets if one), key=len, reverse=True):
+        text = text.replace(secret, REDACTED)
+    return text
+
+
+def tail(
+    payload: bytes,
+    *,
+    count: int = FAILURE_TAIL_LINES,
+    separator: str = " / ",
+    redact: Sequence[str] = (),
+) -> str:
+    """Take the last few lines of a stream, which is what a failure message wants."""
+    return scrub(separator.join(lines_of(payload)[-count:]), redact)
+
+
+async def store(ctx: StepContext, uri: str, payload: bytes) -> None:
+    """Write one captured stream to storage, so it outlives the worker that produced it."""
+    async with ctx.storage.open_write(uri) as sink:
+        await sink.write(payload)
+
+
+def log_stream(ctx: StepContext, stream: str, drained: Drained, redact: Sequence[str] = ()) -> None:
+    """Put what the process printed into the run's log, from both ends of the stream.
+
+    A caller that read the stream through to storage holds only its two ends, and identical
+    ends are how one small enough to be held whole says so. The whole of a stream is elided
+    by line count; what is held in two pieces can only be logged as two pieces.
+
+    A stream that was logged live already has its first lines in the log: this adds the last
+    of them, and only the ones that were never said.
+
+    ``redact`` names the strings a block handed the process that must not reach the log.
+    """
+    log = ctx.log.info if stream == "stdout" else ctx.log.warning
+    half = LOGGED_LINES // 2
+    if drained.logged is not None:
+        remaining = drained.logged.lines - drained.logged.live
+        if remaining <= 0:
+            return
+        last = lines_of(drained.tail)[-min(remaining, half) :]
+        if remaining > len(last):
+            log(f"... {remaining - len(last)} more lines, see the {stream} artifact ...", stream=stream)
+        for line in last:
+            log(scrub(line, redact), stream=stream)
+        return
+    if drained.head == drained.tail:
+        lines = lines_of(drained.head)
+        if not lines:
+            return
+        shown = lines
+        if len(lines) > LOGGED_LINES:
+            shown = [
+                *lines[:half],
+                f"... {len(lines) - LOGGED_LINES} more lines, see the {stream} artifact ...",
+                *lines[-half:],
+            ]
+        for line in shown:
+            log(scrub(line, redact), stream=stream)
+        return
+    first = lines_of(drained.head)
+    last = lines_of(drained.tail)
+    if not first and not last:
+        return
+    for line in first[:half]:
+        log(scrub(line, redact), stream=stream)
+    log(f"... the middle is not in the log, see the {stream} artifact ...", stream=stream)
+    for line in last[-half:]:
+        log(scrub(line, redact), stream=stream)