dirigent-block-execute 0.17.1__py3-none-any.whl

dirigent_block_execute/messages.py

@@ -0,0 +1,183 @@
+"""Every refusal the execute family makes, catalogued under the ``execute`` prefix."""
+
+from dirigent_common import Catalogue
+
+EXECUTE = Catalogue("execute")
+
+TIMED_OUT = EXECUTE.define("timed_out", "{what} did not finish within {seconds}s")
+
+NO_GIT = EXECUTE.define(
+    "no_git",
+    "git is not on the worker's PATH, so nothing here can check a repository out; "
+    "the worker image installs it, a bare host may not",
+)
+
+GIT_EXITED = EXECUTE.define("git_exited", "git {command} exited {code}: {detail}")
+
+CHECKOUT_THROUGH_A_SYMLINK = EXECUTE.define(
+    "checkout_through_a_symlink",
+    "the checkout target {target} leads through the symlink {walked}, which can point "
+    "anywhere; a target is a path of real directories under the run's work directory {base}",
+)
+
+CHECKOUT_OUTSIDE_THE_WORK_DIRECTORY = EXECUTE.define(
+    "checkout_outside_the_work_directory",
+    "the checkout target {target} lands at {destination}, which is outside the run's work directory {base}",
+)
+
+COMPOSE_UP_EXITED = EXECUTE.define("compose_up_exited", "docker compose up exited {code}: {detail}")
+
+COMPOSE_DOWN_EXITED = EXECUTE.define("compose_down_exited", "docker compose down exited {code}: {detail}")
+
+NO_REGISTRY_CREDENTIAL = EXECUTE.define(
+    "no_registry_credential",
+    "docker.build cannot push through a connection with no registry credential: set "
+    "username and password on the docker connection, and registry unless it is Docker Hub",
+)
+
+BUILD_EXITED = EXECUTE.define("build_exited", "docker build exited {code}: {detail}")
+
+NO_IMAGE_ID = EXECUTE.define("no_image_id", "docker build reported success but wrote no image id to the iidfile")
+
+LOGIN_FAILED = EXECUTE.define("login_failed", "docker login to {registry} failed: {detail}")
+
+PUSH_EXITED = EXECUTE.define("push_exited", "docker push {tag} exited {code}: {detail}")
+
+CONTAINER_GONE = EXECUTE.define(
+    "container_gone",
+    "container {container} disappeared before its result could be collected",
+)
+
+OUTPUT_NOT_WRITTEN = EXECUTE.define(
+    "output_not_written",
+    "the container did not write the declared output {name} to {path}",
+)
+
+NO_DAEMON = EXECUTE.define(
+    "no_daemon",
+    "the Docker daemon at {socket} did not answer ({detail}); a worker in a "
+    "container has no daemon unless the host's socket is mounted into it, and "
+    "mounting it grants the container root on the host",
+)
+
+DAEMON_REFUSED = EXECUTE.define("daemon_refused", "the daemon refused to {action}: {detail}")
+
+COMMAND_EXITED = EXECUTE.define("command_exited", "the command exited {code}: {detail}")
+
+
+# What a config refuses at validation. Pydantic owns the code a validator's refusal reaches
+# the wire under, so these are rendered into the ``ValueError`` it wraps.
+
+PUSH_NEEDS_A_CONNECTION = EXECUTE.define(
+    "push_needs_a_connection",
+    "docker.build cannot push without a connection: set connection to a docker connection "
+    "holding registry, username and password",
+)
+
+PUSH_NEEDS_A_TAG = EXECUTE.define(
+    "push_needs_a_tag",
+    "docker.build pushes the tags it built, so a push needs at least one tag",
+)
+
+BUILD_PATHS_STAY_INSIDE = EXECUTE.define(
+    "build_paths_stay_inside",
+    "the context and Dockerfile are paths inside the run's work directory, so they cannot be absolute or climb out",
+)
+
+COMPOSE_UP_ONE_SOURCE = EXECUTE.define(
+    "compose_up_one_source",
+    "docker.compose.up takes either file or content, and exactly one of them",
+)
+
+COMPOSE_DOWN_ONE_SOURCE = EXECUTE.define(
+    "compose_down_one_source",
+    "docker.compose.down takes at most one of file or content",
+)
+
+COMPOSE_PATHS_STAY_INSIDE = EXECUTE.define(
+    "compose_paths_stay_inside",
+    "a compose file or env file is a path inside the run's work directory, so it cannot be absolute or climb out",
+)
+
+GIT_ONE_CREDENTIAL = EXECUTE.define(
+    "git_one_credential",
+    "a git connection carries one credential: a token for an https remote or an ssh_key for an ssh one, never both",
+)
+
+GIT_TOKEN_NEEDS_HTTPS = EXECUTE.define(
+    "git_token_needs_https",
+    "a token is HTTP basic auth, so the url must be http or https, not {url}",
+)
+
+GIT_KEY_NEEDS_SSH = EXECUTE.define("git_key_needs_ssh", "an ssh_key needs an ssh remote, and {url} is not one")
+
+CHECKOUT_STAYS_INSIDE = EXECUTE.define(
+    "checkout_stays_inside",
+    "a checkout target is a path inside the run's work directory, so it cannot be absolute or climb out",
+)
+
+SHELL_ONE_FORM = EXECUTE.define("shell_one_form", "shell.run takes either argv or command, and exactly one of them")
+
+CWD_STAYS_INSIDE = EXECUTE.define(
+    "cwd_stays_inside",
+    "cwd is a path inside the run's work directory, so it cannot be absolute or climb out",
+)
+
+NOT_A_DAEMON_SCHEME = EXECUTE.define(
+    "not_a_daemon_scheme",
+    "a docker host is one of {schemes}, and {host} is none of them",
+)
+
+TLS_IS_ALL_THREE = EXECUTE.define(
+    "tls_is_all_three",
+    "client TLS is all three of tls_ca, tls_cert and tls_key, or none of them",
+)
+
+TLS_NEEDS_TCP = EXECUTE.define(
+    "tls_needs_tcp",
+    "client TLS is how a tcp:// daemon is reached, so the host must be a tcp:// one",
+)
+
+CREDENTIAL_IS_A_PAIR = EXECUTE.define(
+    "credential_is_a_pair",
+    "a registry credential is a username and a password together, never one of them",
+)
+
+REGISTRY_WITHOUT_A_CREDENTIAL = EXECUTE.define(
+    "registry_without_a_credential",
+    "a registry ({registry}) with no username and password authenticates to nothing",
+)
+
+DOCKER_CONNECTION_NAMES_NEITHER = EXECUTE.define(
+    "docker_connection_names_neither",
+    "a docker connection names a daemon, a registry credential, or both, and this names neither",
+)
+
+RUN_ONE_FORM = EXECUTE.define(
+    "run_one_form",
+    "docker.run takes either argv or command, and never both; omit both to run the image's own entrypoint",
+)
+
+RUN_ONE_CPU_FIELD = EXECUTE.define("run_one_cpu_field", "docker.run takes either cpus or nano_cpus, and never both")
+
+RUN_ONE_DAEMON = EXECUTE.define(
+    "run_one_daemon",
+    "a connection names the daemon, so docker.run takes either connection or socket_path",
+)
+
+MOUNTED_FILE_STAYS_INSIDE = EXECUTE.define(
+    "mounted_file_stays_inside",
+    "a mounted file is named inside its mount, so it cannot be absolute or climb out",
+)
+
+OUTPUT_STAYS_INSIDE = EXECUTE.define(
+    "output_stays_inside",
+    "an output without a URI scheme is a path inside the run's work directory, so it cannot be absolute or climb out",
+)
+
+RESERVED_ENVIRONMENT = EXECUTE.define(
+    "reserved_environment",
+    "env_allowlist may not inherit the instance's own configuration: {reserved}. "
+    "{prefix}* holds this instance's secrets, including the envelope key for "
+    "every stored connection; pass what the step needs through env, or a connection.",
+)

dirigent_block_execute/reap.py

@@ -0,0 +1,174 @@
+"""The orphan reaper: compose stacks a run left behind, taken down on the daemon that holds them.
+
+``docker.compose.up`` names its project ``dirigent-<run id>`` and compose labels every container
+of it ``com.docker.compose.project``, so a stack is addressable by the run that created it long
+after that run has gone. A stack is an orphan when its run is terminal, or when no such run
+exists at all, and the project is still up: a worker died mid-run, a pipeline was cancelled
+between ``up`` and ``down``, a ``down`` step never ran.
+
+The reaper sees only the daemon it is pointed at. With one docker-in-docker sidecar per worker
+that is exactly the stacks that worker created, which is the deployment this is written for.
+"""
+
+import re
+from collections.abc import Awaitable, Callable, Mapping
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import NamedTuple
+from uuid import UUID
+
+from dirigent_block_execute import subprocess
+from dirigent_block_execute.capture import tail
+from dirigent_block_execute.compose import PROJECT_LABEL
+from dirigent_block_execute.docker import DockerDaemon, open_client, resolve_endpoint
+from dirigent_plugin import BlockFailure
+
+#: A compose project this instance created, and the run id written into its name.
+DIRIGENT_PROJECT = re.compile(r"^dirigent-([0-9a-f]{32})$")
+
+#: The status a project whose run no longer exists is reported under.
+UNKNOWN_RUN = "unknown"
+
+#: How long one reap invocation of the CLI or the daemon may take.
+REAP_TIMEOUT_SECONDS = 180.0
+
+
+class RunFact(NamedTuple):
+    """What the reaper needs to know about the run a project was named after."""
+
+    status: str
+    """The run's status, as the instance spells it."""
+
+    active: bool
+    """Whether the run may still be driving this stack, in which case it is never touched."""
+
+
+#: Answers what a run id is doing, or ``None`` where the instance holds no such run.
+type RunLookup = Callable[[UUID], Awaitable[RunFact | None]]
+
+
+class Project(NamedTuple):
+    """One compose project on the daemon, with the moment its youngest container was created."""
+
+    name: str
+    run_id: UUID
+    created: datetime
+
+
+class Reaped(NamedTuple):
+    """One project the reaper acted on, or would have acted on."""
+
+    project: str
+    run_id: UUID
+    run_status: str
+    torn_down: bool
+    """False for a dry run, and for a teardown the CLI refused."""
+
+    detail: str = ""
+    """What compose said, when it said anything worth carrying."""
+
+
+def daemon(environ: Mapping[str, str], timeout: float = REAP_TIMEOUT_SECONDS) -> DockerDaemon:
+    """Build the daemon facade the reaper reads the project list through."""
+    endpoint = resolve_endpoint(None, environ)
+    return DockerDaemon(open_client(endpoint, timeout), endpoint.socket)
+
+
+async def projects(client: DockerDaemon) -> list[Project]:
+    """List this instance's compose projects on the daemon, newest container first per project.
+
+    A project's age is its **youngest** container's, so a stack something is still adding to is
+    young however long its first container has been up.
+    """
+    listed = await client.list_labelled({"label": [PROJECT_LABEL]})
+    newest: dict[str, tuple[UUID, datetime]] = {}
+    for container in listed:
+        name = container.labels.get(PROJECT_LABEL, "")
+        found = DIRIGENT_PROJECT.match(name)
+        if found is None:
+            continue
+        created = datetime.fromtimestamp(container.created, tz=UTC)
+        run_id = UUID(found.group(1))
+        held = newest.get(name)
+        if held is None or created > held[1]:
+            newest[name] = (run_id, created)
+    return sorted(
+        (Project(name, run_id, created) for name, (run_id, created) in newest.items()),
+        key=lambda project: project.name,
+    )
+
+
+async def orphans(
+    found: list[Project], look_up: RunLookup, *, grace: timedelta, now: datetime | None = None
+) -> list[tuple[Project, str]]:
+    """Choose the projects to reap, and say which status each was chosen for.
+
+    A project younger than the grace is left alone whatever its run says, because a stack whose
+    ``up`` has only just returned is a stack whose ``down`` has not been reached yet. A run that
+    is still active is never reaped; one that is terminal, and one the instance no longer holds,
+    both are.
+    """
+    moment = now or datetime.now(UTC)
+    chosen: list[tuple[Project, str]] = []
+    for project in found:
+        if moment - project.created < grace:
+            continue
+        fact = await look_up(project.run_id)
+        if fact is None:
+            chosen.append((project, UNKNOWN_RUN))
+        elif not fact.active:
+            chosen.append((project, fact.status))
+    return chosen
+
+
+async def reap(
+    environ: Mapping[str, str],
+    root: Path,
+    look_up: RunLookup,
+    *,
+    grace: timedelta,
+    dry_run: bool = False,
+    command_path: list[str] | None = None,
+    now: datetime | None = None,
+) -> list[Reaped]:
+    """Run one pass: list the projects, choose the orphans, and take each one down.
+
+    A teardown that fails is reported rather than raised, so one stack the daemon will not let
+    go of does not stop the pass reaching the next.
+    """
+    async with daemon(environ) as client:
+        found = await projects(client)
+    chosen = await orphans(found, look_up, grace=grace, now=now)
+    argv_head = command_path or ["docker", "compose"]
+    results: list[Reaped] = []
+    for project, status in chosen:
+        if dry_run:
+            results.append(Reaped(project.name, project.run_id, status, torn_down=False))
+            continue
+        results.append(await _tear_down(argv_head, project, status, environ, root))
+    return results
+
+
+async def _tear_down(
+    command_path: list[str], project: Project, status: str, environ: Mapping[str, str], root: Path
+) -> Reaped:
+    """Take one orphaned project down, volumes and orphans included, and say how it went."""
+    try:
+        code, out, err = await subprocess.output(
+            argv=[*command_path, "-p", project.name, "down", "-v", "--remove-orphans"],
+            directory=root,
+            environ=dict(environ),
+            timeout_seconds=REAP_TIMEOUT_SECONDS,
+            what="docker compose down",
+        )
+    except BlockFailure as error:
+        return Reaped(project.name, project.run_id, status, torn_down=False, detail=str(error))
+    if code != 0:
+        return Reaped(
+            project.name,
+            project.run_id,
+            status,
+            torn_down=False,
+            detail=tail(err) or tail(out) or f"docker compose down exited {code}",
+        )
+    return Reaped(project.name, project.run_id, status, torn_down=True)

dirigent_block_execute/secrets.py

@@ -0,0 +1,42 @@
+"""A private directory for the credential files a CLI reads, and the writes that fill it.
+
+A tool that takes a credential takes it from a file: git through an askpass helper, docker
+through a client key and a config directory. The file is 0600 inside a 0700 directory, is
+created with those permissions rather than fixed after, and goes away when the step leaves
+however it leaves.
+"""
+
+import contextlib
+import os
+import shutil
+import stat
+import tempfile
+from collections.abc import Generator
+from pathlib import Path
+
+#: A file only its owner may read or write.
+PRIVATE = stat.S_IRUSR | stat.S_IWUSR
+
+#: A file only its owner may read, write or run.
+EXECUTABLE = stat.S_IRWXU
+
+
+@contextlib.contextmanager
+def private_directory(parent: Path, prefix: str) -> Generator[Path]:
+    """Make a 0700 directory under a parent, and remove it and everything in it afterwards."""
+    directory = Path(tempfile.mkdtemp(prefix=prefix, dir=parent))
+    directory.chmod(stat.S_IRWXU)
+    try:
+        yield directory
+    finally:
+        shutil.rmtree(directory, ignore_errors=True)
+
+
+def write(path: Path, text: str, mode: int = PRIVATE) -> None:
+    """Write one credential file, created with its final permissions rather than fixed after."""
+    handle = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
+    with os.fdopen(handle, "w") as sink:
+        sink.write(text)
+
+
+__all__ = ["EXECUTABLE", "PRIVATE", "private_directory", "write"]

dirigent_block_execute/shell.py

@@ -0,0 +1,174 @@
+"""``shell.run``: the one built-in block that executes code on the worker.
+
+It declares ``local_execution``, which means the engine refuses to run it at all unless the
+instance config allowlists its id. That gate exists because "can edit pipelines" must never
+silently mean "can run code on workers".
+"""
+
+from datetime import timedelta
+from pathlib import Path
+from typing import Annotated, ClassVar
+
+from pydantic import BaseModel, Field, model_validator
+
+from dirigent_block_execute import subprocess
+from dirigent_block_execute.capture import log_stream, tail
+from dirigent_block_execute.environment import reject_reserved
+from dirigent_block_execute.messages import (
+    COMMAND_EXITED,
+    CWD_STAYS_INSIDE,
+    SHELL_ONE_FORM,
+)
+from dirigent_common import BlockModel, Duration
+from dirigent_plugin import (
+    BlockFailure,
+    ErrorClass,
+    Operator,
+    OperatorSpec,
+    RemoteHandle,
+    ShellString,
+    ShellVariables,
+    StepContext,
+)
+
+
+class ShellRunConfig(ShellVariables):
+    """What to run, where, with what environment, and for how long."""
+
+    argv: list[str] = Field(default_factory=list[str])
+    """The command as an argument vector, which does not involve a shell."""
+
+    command: Annotated[str | None, ShellString()] = None
+    """The command as a shell string, for when a pipe or a redirect is the point.
+
+    Every ``${...}`` in it is rewritten by the engine to a variable it sets in this command's
+    environment, so a value that came from a webhook payload is one word the shell never
+    parses, however the reference was quoted -- and inside single quotes, which a shell keeps
+    literal, the command reads that variable's name rather than its value. Prefer ``argv``
+    anyway: it involves no shell at all."""
+
+    cwd: str | None = None
+    """A directory relative to the run's work directory; never an absolute path."""
+
+    env: dict[str, str] = Field(default_factory=dict[str, str])
+    """Variables set explicitly for this command."""
+
+    env_allowlist: list[str] = Field(default_factory=list[str])
+    """Worker environment variables this command is allowed to inherit.
+
+    Never the instance's own ``DIRIGENT_*`` variables: those hold this instance's secrets,
+    and a step that could inherit them could print the envelope key into the run log."""
+
+    timeout: Duration = Field(default=timedelta(minutes=5), gt=timedelta(0))
+    """How long the process may run before it is killed as a transient failure."""
+
+    @model_validator(mode="after")
+    def _require_one_form(self) -> "ShellRunConfig":
+        """Reject a config that names both forms of the command, or neither."""
+        if bool(self.argv) == bool(self.command):
+            raise ValueError(SHELL_ONE_FORM.render())
+        if self.cwd and (Path(self.cwd).is_absolute() or ".." in Path(self.cwd).parts):
+            raise ValueError(CWD_STAYS_INSIDE.render())
+        reject_reserved(self.env_allowlist)
+        return self
+
+
+class ShellRunOutput(BlockModel):
+    """What the command did, with its full streams addressable as artifacts."""
+
+    exit_code: int
+
+    stdout: str
+    """The head of what the command printed, cut at the instance's inline capture size.
+
+    Verbatim, including the trailing newline a command like ``echo`` ends with. Use
+    ``printf '%s'`` where the value is read by another step."""
+
+    stderr: str
+    """The head of what the command printed to stderr, cut the same way."""
+
+    stdout_uri: str
+    """Where the whole of stdout was written; never truncated, whatever the field above holds."""
+
+    stderr_uri: str
+    """Where the whole of stderr was written; never truncated either."""
+
+    stdout_bytes: int
+    """How much the command printed to stdout altogether, inlined or not."""
+
+    stderr_bytes: int
+    """How much it printed to stderr altogether."""
+
+    stdout_truncated: bool
+    """Whether ``stdout`` above is short of the stream. Only the inline copy is ever cut."""
+
+    stderr_truncated: bool
+    """Whether ``stderr`` above is short of the stream, which is an independent question."""
+
+
+def _environment(config: ShellRunConfig, home: Path) -> dict[str, str]:
+    """Build the command's environment, the substituted values last.
+
+    They are set after ``env`` so a document cannot override what a reference resolved to,
+    and they are set whatever the allowlist holds: the command reads them because the engine
+    put them in it.
+    """
+    environ = subprocess.environment(config.env_allowlist, config.env, home)
+    environ.update(config.shell_variables)
+    return environ
+
+
+class ShellRunOperator(Operator[ShellRunConfig, ShellRunOutput]):
+    """Runs a command on the worker, inside the run's work directory and behind the allowlist."""
+
+    spec = OperatorSpec(
+        id="shell.run",
+        group="execute",
+        summary="Run a command on the worker.",
+        idempotent=False,
+        local_execution=True,
+    )
+    config_model: ClassVar[type[BaseModel]] = ShellRunConfig
+    output_model: ClassVar[type[BaseModel]] = ShellRunOutput
+
+    async def execute(self, config: ShellRunConfig, ctx: StepContext) -> ShellRunOutput | RemoteHandle:
+        """Run the command once, capture both streams, and classify how it ended."""
+        workspace = subprocess.workspace(ctx, "shell")
+        directory = workspace / config.cwd if config.cwd else workspace
+        directory.mkdir(parents=True, exist_ok=True)
+        artifacts = subprocess.prefix(ctx, "shell")
+        stdout_uri = f"{artifacts}-stdout.txt"
+        stderr_uri = f"{artifacts}-stderr.txt"
+        code, out, err = await subprocess.run(
+            directory=directory,
+            ctx=ctx,
+            stdout_uri=stdout_uri,
+            stderr_uri=stderr_uri,
+            timeout_seconds=config.timeout.total_seconds(),
+            environ=_environment(config, directory),
+            argv=config.argv or None,
+            command=config.command,
+        )
+        log_stream(ctx, "stdout", out)
+        log_stream(ctx, "stderr", err)
+        ctx.log.info("command finished", exit_code=code, stdout_bytes=out.total_bytes, stderr_bytes=err.total_bytes)
+        if code != 0:
+            raise BlockFailure(
+                COMMAND_EXITED,
+                error_class=ErrorClass.UNKNOWN,
+                code=code,
+                detail=tail(err.tail) or tail(out.tail) or "no output",
+            )
+        printed = out.captured(ctx.inline_capture)
+        failed = err.captured(ctx.inline_capture)
+        return ShellRunOutput(
+            exit_code=code,
+            stdout=printed.text,
+            stderr=failed.text,
+            stdout_uri=stdout_uri,
+            stderr_uri=stderr_uri,
+            stdout_bytes=printed.total_bytes,
+            stderr_bytes=failed.total_bytes,
+            stdout_truncated=printed.truncated,
+            stderr_truncated=failed.truncated,
+        )