djaploy 1.6.0__tar.gz → 1.6.2__tar.gz

{djaploy-1.6.0 → djaploy-1.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: djaploy
-Version: 1.6.0
+Version: 1.6.2
 Summary: Modular Django deployment system based on pyinfra
 Author-email: Johanna Mae Dimayuga <johanna@techco.fi>
 License: MIT License
@@ -318,11 +318,11 @@ cluster. The inventory host is the **build node**.
 
 ```python
 HostConfig(
-    "extor",                                  # build node: a cluster node with BuildKit
-    ssh_hostname="65.109.151.140",
+    "monster",                                # build node: runs the BuildKit service
+    ssh_hostname="167.235.97.139",
     ssh_user="janitor",                       # needs passwordless sudo
     ssh_key=OpFilePath("/Infra/SSH Key/private key?ssh-format=openssh"),
-    ssh_known_hosts_file=OpFilePath("/Infra/extor/SSH Identity"),
+    ssh_known_hosts_file="~/src/hetzner-management/servers/monster/known_hosts-167.235.97.139",
     deployment_strategy="gitops",
     app_name="docms",
     manage_py_path="docms/manage.py",
@@ -344,11 +344,15 @@ HostConfig(
 `deploy` (all from the deploying machine, after the artifact is created):
 
 1. checks the GitOps checkout has no local edits to the manifest and pulls it;
-2. builds the artifact on the build node with a temporary BuildKit daemon
-   (`/opt/techco-buildkit`, root-only socket, cache under `/data/djaploy-build`)
-   and pushes `<image>:<commit>` using a one-hour token for the namespace's
+2. builds the artifact with the build node's BuildKit and pushes
+   `<image>:<commit>` using a one-hour token for the namespace's
    `registry-builder` service account; build args `MANAGE_PY` and
-   `SETTINGS_MODULE`, as for k3s;
+   `SETTINGS_MODULE`, as for k3s. By default (`builder.mode="shared"`) the job
+   runs as the `builder` user against the node's always-running rootless
+   BuildKit service (`/run/techco-buildkit/buildkitd.sock`, jobs under
+   `/var/lib/techco-buildkit/jobs`, shared cache); `builder.mode="temporary"`
+   instead starts a root BuildKit daemon for the one build (djaploy 1.6.0's
+   behaviour). The job directory and credentials are removed afterwards;
 3. applies the env Secret (`secret_key` + scalar `data`), so 1Password stays
    the source of secrets and none are in Git;
 4. rewrites the digest pin in the manifest (a kustomize `images:` entry or

{djaploy-1.6.0 → djaploy-1.6.2}/README.md

@@ -256,11 +256,11 @@ cluster. The inventory host is the **build node**.
 
 ```python
 HostConfig(
-    "extor",                                  # build node: a cluster node with BuildKit
-    ssh_hostname="65.109.151.140",
+    "monster",                                # build node: runs the BuildKit service
+    ssh_hostname="167.235.97.139",
     ssh_user="janitor",                       # needs passwordless sudo
     ssh_key=OpFilePath("/Infra/SSH Key/private key?ssh-format=openssh"),
-    ssh_known_hosts_file=OpFilePath("/Infra/extor/SSH Identity"),
+    ssh_known_hosts_file="~/src/hetzner-management/servers/monster/known_hosts-167.235.97.139",
     deployment_strategy="gitops",
     app_name="docms",
     manage_py_path="docms/manage.py",
@@ -282,11 +282,15 @@ HostConfig(
 `deploy` (all from the deploying machine, after the artifact is created):
 
 1. checks the GitOps checkout has no local edits to the manifest and pulls it;
-2. builds the artifact on the build node with a temporary BuildKit daemon
-   (`/opt/techco-buildkit`, root-only socket, cache under `/data/djaploy-build`)
-   and pushes `<image>:<commit>` using a one-hour token for the namespace's
+2. builds the artifact with the build node's BuildKit and pushes
+   `<image>:<commit>` using a one-hour token for the namespace's
    `registry-builder` service account; build args `MANAGE_PY` and
-   `SETTINGS_MODULE`, as for k3s;
+   `SETTINGS_MODULE`, as for k3s. By default (`builder.mode="shared"`) the job
+   runs as the `builder` user against the node's always-running rootless
+   BuildKit service (`/run/techco-buildkit/buildkitd.sock`, jobs under
+   `/var/lib/techco-buildkit/jobs`, shared cache); `builder.mode="temporary"`
+   instead starts a root BuildKit daemon for the one build (djaploy 1.6.0's
+   behaviour). The job directory and credentials are removed afterwards;
 3. applies the env Secret (`secret_key` + scalar `data`), so 1Password stays
    the source of secrets and none are in Git;
 4. rewrites the digest pin in the manifest (a kustomize `images:` entry or

{djaploy-1.6.0 → djaploy-1.6.2}/djaploy/deploy.py

@@ -61,7 +61,15 @@ def run_command(context: Dict[str, Any]) -> None:
     call_hook(f"{command_name}:precommand", context)
     call_hook("precommand", context)
 
-    # 2. Run pyinfra
+    # 2. Run pyinfra (unless a precommand hook already did all the work,
+    #    e.g. the gitops strategy, which has no remote hooks)
+    if context.get("skip_remote"):
+        print(f"No remote execution needed ({context['skip_remote']}).", flush=True)
+        context["success"] = True
+        call_hook(f"{command_name}:postcommand", context)
+        call_hook("postcommand", context)
+        return
+
     print("Starting remote execution...", flush=True)
     processed_inventory = _preprocess_inventory(str(context["inventory_file"]))
 

{djaploy-1.6.0 → djaploy-1.6.2}/djaploy/infra/gitops.py

@@ -7,15 +7,17 @@ pins its digest in the GitOps repository and syncs the Argo CD application:
 
     deploy:precommand (local, after the artifact is created)
       1. check the GitOps checkout is clean and up to date
-      2. build the artifact on the build node with a temporary BuildKit daemon
-         and push it with a one-hour registry token for the app's namespace
+      2. build the artifact with the build node's BuildKit (its shared rootless
+         service, or a temporary daemon) and push it with a one-hour registry
+         token for the app's namespace
       3. apply the env Secret (SECRET_KEY + data, e.g. from OpSecret)
       4. pin the pushed digest in the GitOps repository, commit and push
       5. sync that commit in Argo CD and wait until Synced and Healthy
 
 The inventory host is the build node: its SSH connection (ssh_hostname,
-ssh_user, ssh_key, ssh_known_hosts_file) is used for the build, and no remote
-pyinfra hooks run. Kubernetes access uses the local kubeconfig.
+ssh_user, ssh_key, ssh_known_hosts_file) is used for the build over one
+multiplexed connection, retried if the connection drops. pyinfra is not run
+for gitops-only inventories. Kubernetes access uses the local kubeconfig.
 
 HostConfig fields used: app_name, manage_py_path, secret_key, data, and
 ``gitops_conf``::
@@ -29,9 +31,11 @@ HostConfig fields used: app_name, manage_py_path, secret_key, data, and
         "kubeconfig": "~/.kube/techco-me.kubeconfig",    # default: $KUBECONFIG
         "env_secret": "docms-env",                       # default: <app_name>-env; None: leave Secrets alone
         "settings_module": "docms.settings.production",  # Dockerfile build arg
-        "builder": {                                     # build node layout (defaults shown)
-            "buildkit_dir": "/opt/techco-buildkit",
-            "work_dir": "/data/djaploy-build",
+        "builder": {                                     # defaults shown
+            "mode": "shared",                            # node's BuildKit service; or "temporary"
+            "user": "builder",
+            "socket": "unix:///run/techco-buildkit/buildkitd.sock",
+            "work_dir": "/var/lib/techco-buildkit/jobs",
         },
         "registry_audience": "registry.techco.fi",
         "sync_timeout": 900,                             # seconds to wait for Argo CD
@@ -47,7 +51,9 @@ import json
 import os
 import re
 import shlex
+import shutil
 import subprocess
+import tempfile
 import time
 import uuid
 from pathlib import Path
@@ -200,11 +206,23 @@ def check_infra_repo(repo: Path, manifest: str) -> str:
 
 
 class BuildNode:
-    """SSH to the build node with the inventory's connection settings."""
+    """SSH to the build node with the inventory's connection settings.
+
+    All commands share one multiplexed connection, so a build makes one SSH
+    handshake. A failed connection (ssh exit status 255) is retried: every
+    build-job command is safe to repeat, and an input stream is rewound first.
+    """
+
+    RETRIES = 3
 
     def __init__(self, host_data):
         self.target = f"{_get(host_data, 'ssh_user')}@{_get(host_data, 'ssh_hostname')}"
-        self.opts = ["-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes"]
+        # Short private directory: Unix socket paths are limited (~104 bytes on macOS).
+        self.control_dir = tempfile.mkdtemp(prefix="djaploy-ssh-", dir="/tmp")
+        self.opts = ["-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes",
+                     "-o", "ConnectTimeout=20", "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=4",
+                     "-o", "ControlMaster=auto", "-o", f"ControlPath={self.control_dir}/%C",
+                     "-o", "ControlPersist=120"]
         if _get(host_data, "ssh_port"):
             self.opts += ["-p", str(_get(host_data, "ssh_port"))]
         if _get(host_data, "ssh_key"):
@@ -213,22 +231,88 @@ class BuildNode:
             self.opts += ["-o", f"UserKnownHostsFile={_get(host_data, 'ssh_known_hosts_file')}"]
 
     def run(self, command: str, **kwargs):
-        return _run(["ssh", *self.opts, self.target, command], **kwargs)
+        stdin = kwargs.get("stdin")
+        for attempt in range(1, self.RETRIES + 1):
+            if attempt > 1 and hasattr(stdin, "seek"):
+                stdin.seek(0)
+            try:
+                return _run(["ssh", *self.opts, self.target, command], **kwargs)
+            except subprocess.CalledProcessError as exc:
+                if exc.returncode != 255 or attempt == self.RETRIES:
+                    raise
+                print(f"[gitops] SSH connection to {self.target} failed; retrying ({attempt}/{self.RETRIES - 1})",
+                      flush=True)
+                time.sleep(2 * attempt)
+
+    def close(self):
+        _run(["ssh", *self.opts, "-O", "exit", self.target], check=False,
+             stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        shutil.rmtree(self.control_dir, ignore_errors=True)
+
+
+BUILDER_DEFAULTS = {
+    # The build node's always-running rootless BuildKit service (hetzner-management
+    # kubernetes/builder on Monster): jobs run as its user against its socket.
+    "shared": {"user": "builder", "buildkit_dir": "/opt/techco-buildkit",
+               "socket": "unix:///run/techco-buildkit/buildkitd.sock",
+               "work_dir": "/var/lib/techco-buildkit/jobs"},
+    # A root BuildKit daemon started for each build (systemd-run), as djaploy 1.6.0 did.
+    "temporary": {"buildkit_dir": "/opt/techco-buildkit", "work_dir": "/data/djaploy-build"},
+}
+
+
+def builder_conf(host_data) -> dict:
+    given = dict(_get(conf_of(host_data), "builder", {}))
+    mode = given.get("mode", "shared")
+    if mode not in BUILDER_DEFAULTS:
+        raise ValueError(f'gitops_conf["builder"]["mode"] must be "shared" or "temporary", not {mode!r}')
+    return {"mode": mode, **BUILDER_DEFAULTS[mode], **given}
+
+
+def build_commands(builder: dict, work: str, run_id: str, ns: str) -> dict:
+    """Shell commands for one build job on the build node."""
+    q = shlex.quote
+    buildkit = builder["buildkit_dir"]
+    if builder["mode"] == "shared":
+        as_user = f"sudo -u {q(builder['user'])}"
+        sock = builder["socket"]
+        return {
+            "prepare": f"{as_user} mkdir -p {work}/source {work}/auth",
+            "extract": f"{as_user} tar --no-same-owner -xzf - -C {work}/source",
+            "auth": f"{as_user} sh -c {q(f'umask 077; cat > {work}/auth/config.json')}",
+            "start": None,
+            "ready": None,
+            "build": f"{as_user} env DOCKER_CONFIG={work}/auth {buildkit}/buildctl --addr {q(sock)} build",
+            "result": f"{as_user} cat {work}/result.json",
+            "cleanup": f"{as_user} rm -rf -- {work}",
+        }
+    unit = f"djaploy-buildkit-{run_id}"
+    sock = f"unix:///run/{unit}/buildkitd.sock"
+    cache = f"{builder['work_dir']}/cache-{ns}"
+    return {
+        "prepare": f"sudo install -d -m 700 {work}/source {work}/auth {cache} /run/{unit}",
+        "extract": f"sudo tar -xzf - -C {work}/source",
+        "auth": f"sudo sh -c {q(f'umask 077; cat > {work}/auth/config.json')}",
+        "start": (f"sudo systemd-run --collect --unit={unit} --property=UMask=0077 {buildkit}/buildkitd "
+                  f"--addr {sock} --root {cache} --oci-worker=true --containerd-worker=false "
+                  f"--oci-worker-binary {buildkit}/buildkit-runc"),
+        "ready": f"sudo {buildkit}/buildctl --addr {sock} debug workers",
+        "build": f"sudo env DOCKER_CONFIG={work}/auth {buildkit}/buildctl --addr {sock} build",
+        "result": f"sudo cat {work}/result.json",
+        "cleanup": f"sudo systemctl stop {unit} 2>/dev/null; sudo rm -rf -- {work}",
+    }
 
 
 def build_and_push(host_data, artifact_path: Path, image: str, tag: str) -> str:
     """Build on the build node and push; return the pushed manifest digest."""
     conf = conf_of(host_data)
     ns = namespace(host_data)
-    builder = dict(_get(conf, "builder", {}))
-    buildkit = builder.get("buildkit_dir", "/opt/techco-buildkit")
+    builder = builder_conf(host_data)
     run_id = f"{ns}-{tag}-{uuid.uuid4().hex[:8]}"
-    work = f"{builder.get('work_dir', '/data/djaploy-build')}/{run_id}"
-    cache = f"{builder.get('work_dir', '/data/djaploy-build')}/cache-{ns}"
-    unit = f"djaploy-buildkit-{run_id}"
-    sock = f"unix:///run/{unit}/buildkitd.sock"
     if not re.fullmatch(r"[a-z0-9][a-z0-9.-]*", run_id):
         raise ValueError(f"Unsafe build id {run_id!r}")
+    work = f"{builder['work_dir']}/{run_id}"
+    cmd = build_commands(builder, work, run_id, ns)
     node = BuildNode(host_data)
 
     token = _run(kubectl(host_data) + ["-n", ns, "create", "token", "registry-builder",
@@ -242,31 +326,29 @@ def build_and_push(host_data, artifact_path: Path, image: str, tag: str) -> str:
     if _get(conf, "settings_module"):
         args.append(f"--opt build-arg:SETTINGS_MODULE={shlex.quote(conf['settings_module'])}")
 
-    print(f"[gitops] Building {image}:{tag} on {node.target}", flush=True)
-    node.run(f"sudo install -d -m 700 {work}/source {work}/auth {cache} /run/{unit}")
+    print(f"[gitops] Building {image}:{tag} on {node.target} ({builder['mode']} BuildKit)", flush=True)
+    node.run(cmd["prepare"])
     try:
         with open(artifact_path, "rb") as archive:
-            node.run(f"sudo tar -xzf - -C {work}/source", stdin=archive)
-        node.run(f"sudo tee {work}/auth/config.json >/dev/null", input=docker_config, text=True,
-                 stdout=subprocess.DEVNULL)
-        node.run(f"sudo systemd-run --collect --unit={unit} --property=UMask=0077 {buildkit}/buildkitd "
-                 f"--addr {sock} --root {cache} --oci-worker=true --containerd-worker=false "
-                 f"--oci-worker-binary {buildkit}/buildkit-runc", stdout=subprocess.DEVNULL)
-        for _ in range(60):
-            ready = node.run(f"sudo {buildkit}/buildctl --addr {sock} debug workers",
-                             check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-            if ready.returncode == 0:
-                break
-            time.sleep(0.5)
-        else:
-            raise RuntimeError("BuildKit did not become ready on the build node")
-        node.run(f"sudo env DOCKER_CONFIG={work}/auth {buildkit}/buildctl --addr {sock} build --progress=plain "
-                 f"--frontend dockerfile.v0 --local context={work}/source --local dockerfile={work}/source "
+            node.run(cmd["extract"], stdin=archive)
+        node.run(cmd["auth"], input=docker_config, text=True, stdout=subprocess.DEVNULL)
+        if cmd["start"]:
+            node.run(cmd["start"], stdout=subprocess.DEVNULL)
+            for _ in range(60):
+                ready = node.run(cmd["ready"], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                if ready.returncode == 0:
+                    break
+                time.sleep(0.5)
+            else:
+                raise RuntimeError("BuildKit did not become ready on the build node")
+        node.run(f"{cmd['build']} --progress=plain --frontend dockerfile.v0 "
+                 f"--local context={work}/source --local dockerfile={work}/source "
                  f"{' '.join(args)} --output type=image,name={image}:{tag},push=true "
                  f"--metadata-file {work}/result.json")
-        result = json.loads(node.run(f"sudo cat {work}/result.json", capture_output=True, text=True).stdout)
+        result = json.loads(node.run(cmd["result"], capture_output=True, text=True).stdout)
     finally:
-        node.run(f"sudo systemctl stop {unit} 2>/dev/null; sudo rm -rf {work}", check=False)
+        node.run(cmd["cleanup"], check=False)
+        node.close()
     digest = result.get("containerimage.digest", "")
     if not DIGEST_RE.fullmatch(digest):
         raise RuntimeError(f"Build finished without a pushed digest: {result}")
@@ -349,6 +431,9 @@ def _gitops_deploy(context):
     gitops_hosts = [data for _, data in hosts if _get(data, "deployment_strategy") == GITOPS]
     if not gitops_hosts:
         return
+    if len(gitops_hosts) == len(hosts):
+        # No remote pyinfra hooks run for gitops hosts: don't connect at all.
+        context["skip_remote"] = "gitops deploys from this machine"
     if len(gitops_hosts) > 1:
         raise ValueError("gitops inventories deploy one app per environment; use one host")
     host_data = gitops_hosts[0]

djaploy-1.6.2/djaploy/version.py

@@ -0,0 +1 @@
+__version__ = "1.6.2"

{djaploy-1.6.0 → djaploy-1.6.2}/djaploy.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: djaploy
-Version: 1.6.0
+Version: 1.6.2
 Summary: Modular Django deployment system based on pyinfra
 Author-email: Johanna Mae Dimayuga <johanna@techco.fi>
 License: MIT License
@@ -318,11 +318,11 @@ cluster. The inventory host is the **build node**.
 
 ```python
 HostConfig(
-    "extor",                                  # build node: a cluster node with BuildKit
-    ssh_hostname="65.109.151.140",
+    "monster",                                # build node: runs the BuildKit service
+    ssh_hostname="167.235.97.139",
     ssh_user="janitor",                       # needs passwordless sudo
     ssh_key=OpFilePath("/Infra/SSH Key/private key?ssh-format=openssh"),
-    ssh_known_hosts_file=OpFilePath("/Infra/extor/SSH Identity"),
+    ssh_known_hosts_file="~/src/hetzner-management/servers/monster/known_hosts-167.235.97.139",
     deployment_strategy="gitops",
     app_name="docms",
     manage_py_path="docms/manage.py",
@@ -344,11 +344,15 @@ HostConfig(
 `deploy` (all from the deploying machine, after the artifact is created):
 
 1. checks the GitOps checkout has no local edits to the manifest and pulls it;
-2. builds the artifact on the build node with a temporary BuildKit daemon
-   (`/opt/techco-buildkit`, root-only socket, cache under `/data/djaploy-build`)
-   and pushes `<image>:<commit>` using a one-hour token for the namespace's
+2. builds the artifact with the build node's BuildKit and pushes
+   `<image>:<commit>` using a one-hour token for the namespace's
    `registry-builder` service account; build args `MANAGE_PY` and
-   `SETTINGS_MODULE`, as for k3s;
+   `SETTINGS_MODULE`, as for k3s. By default (`builder.mode="shared"`) the job
+   runs as the `builder` user against the node's always-running rootless
+   BuildKit service (`/run/techco-buildkit/buildkitd.sock`, jobs under
+   `/var/lib/techco-buildkit/jobs`, shared cache); `builder.mode="temporary"`
+   instead starts a root BuildKit daemon for the one build (djaploy 1.6.0's
+   behaviour). The job directory and credentials are removed afterwards;
 3. applies the env Secret (`secret_key` + scalar `data`), so 1Password stays
    the source of secrets and none are in Git;
 4. rewrites the digest pin in the manifest (a kustomize `images:` entry or

{djaploy-1.6.0 → djaploy-1.6.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "djaploy"
-version = "1.6.0"
+version = "1.6.2"
 description = "Modular Django deployment system based on pyinfra"
 authors = [
     {name = "Johanna Mae Dimayuga", email = "johanna@techco.fi"}

{djaploy-1.6.0 → djaploy-1.6.2}/tests/test_gitops.py

@@ -181,6 +181,7 @@ class SyncOperationTests(unittest.TestCase):
 class BuildNodeTests(unittest.TestCase):
     def test_ssh_options_from_inventory(self):
         node = gitops.BuildNode(host(ssh_port=2222, ssh_known_hosts_file="/tmp/kh"))
+        self.addCleanup(lambda: __import__("shutil").rmtree(node.control_dir, ignore_errors=True))
         self.assertEqual(node.target, "janitor@203.0.113.10")
         self.assertIn("StrictHostKeyChecking=yes", node.opts)
         self.assertIn("UserKnownHostsFile=/tmp/kh", node.opts)
@@ -188,6 +189,124 @@ class BuildNodeTests(unittest.TestCase):
         self.assertEqual(node.opts[node.opts.index("-i") + 1], "/tmp/key")
 
 
+class FakeNode:
+    """Records build-node commands; answers result.json with a digest."""
+
+    def __init__(self, fail_on=None):
+        self.commands, self.fail_on, self.target = [], fail_on, "janitor@node"
+
+    def run(self, command, **kwargs):
+        self.commands.append(command)
+        if self.fail_on and self.fail_on in command:
+            raise subprocess.CalledProcessError(1, command)
+        stdout = '{"containerimage.digest": "%s"}' % NEW if command.endswith("result.json") else ""
+        return subprocess.CompletedProcess(command, 0, stdout=stdout)
+
+    def close(self):
+        self.commands.append("<close>")
+
+
+class BuilderTests(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False)
+        tmp.close()
+        self.addCleanup(Path(tmp.name).unlink)
+        self.artifact = Path(tmp.name)
+
+    def build(self, node, **builder):
+        conf = {**host()["gitops_conf"], **({"builder": builder} if builder else {})}
+        token = subprocess.CompletedProcess([], 0, stdout="tok\n")
+        with patch.object(gitops, "BuildNode", return_value=node), patch.object(gitops, "_run", return_value=token):
+            return gitops.build_and_push(host(gitops_conf=conf), self.artifact, IMAGE, "abc1234")
+
+    def test_defaults_to_shared_builder(self):
+        b = gitops.builder_conf(host())
+        self.assertEqual(b["mode"], "shared")
+        self.assertEqual(b["user"], "builder")
+        self.assertEqual(b["socket"], "unix:///run/techco-buildkit/buildkitd.sock")
+
+    def test_invalid_mode(self):
+        with self.assertRaisesRegex(ValueError, "shared"):
+            gitops.builder_conf(host(gitops_conf={**host()["gitops_conf"], "builder": {"mode": "docker"}}))
+
+    def test_shared_build_sequence(self):
+        node = FakeNode()
+        self.assertEqual(self.build(node), NEW)
+        self.assertEqual(node.commands[-1], "<close>")
+        self.assertTrue(all(c.startswith("sudo -u builder ") for c in node.commands[:-1]), node.commands)
+        self.assertFalse(any("systemd-run" in c for c in node.commands))
+        steps = ["mkdir -p", "tar --no-same-owner", "umask 077", "buildctl --addr", "/result.json", "rm -rf --"]
+        positions = [next(i for i, c in enumerate(node.commands) if step in c and (step != "/result.json" or c.endswith(step))) for step in steps]
+        self.assertEqual(positions, sorted(positions))
+        build = next(c for c in node.commands if " build " in c)
+        self.assertIn("/var/lib/techco-buildkit/jobs/docms-prod-abc1234-", build)
+        self.assertIn(f"name={IMAGE}:abc1234,push=true", build)
+        self.assertIn("build-arg:SETTINGS_MODULE=docms.settings.production", build)
+
+    def test_shared_cleanup_after_failed_build(self):
+        node = FakeNode(fail_on=" build ")
+        with self.assertRaises(subprocess.CalledProcessError):
+            self.build(node)
+        self.assertTrue(node.commands[-2].startswith("sudo -u builder rm -rf -- /var/lib/techco-buildkit/jobs/"))
+        self.assertEqual(node.commands[-1], "<close>")
+
+    def test_temporary_mode_starts_its_own_daemon(self):
+        node = FakeNode()
+        self.assertEqual(self.build(node, mode="temporary"), NEW)
+        self.assertTrue(any("systemd-run" in c for c in node.commands))
+        self.assertTrue(any("/data/djaploy-build/docms-prod-abc1234-" in c for c in node.commands))
+        self.assertIn("systemctl stop djaploy-buildkit-", node.commands[-2])
+
+
+class BuildNodeConnectionTests(unittest.TestCase):
+    def setUp(self):
+        self.node = gitops.BuildNode(host())
+        self.addCleanup(lambda: __import__("shutil").rmtree(self.node.control_dir, ignore_errors=True))
+        patcher = patch.object(gitops.time, "sleep")
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_one_multiplexed_connection_with_keepalives(self):
+        self.assertTrue(self.node.control_dir.startswith("/tmp/djaploy-ssh-"))
+        self.assertIn(f"ControlPath={self.node.control_dir}/%C", self.node.opts)
+        for opt in ("ControlMaster=auto", "ServerAliveInterval=15", "ConnectTimeout=20"):
+            self.assertIn(opt, self.node.opts)
+
+    def test_connection_failure_is_retried_and_input_rewound(self):
+        failures = [subprocess.CalledProcessError(255, "ssh")]
+        seen = []
+
+        def fake_run(cmd, **kwargs):
+            seen.append(kwargs["stdin"].read())
+            if failures:
+                raise failures.pop()
+            return subprocess.CompletedProcess(cmd, 0)
+
+        with tempfile.TemporaryFile() as archive, patch.object(gitops, "_run", side_effect=fake_run):
+            archive.write(b"source")
+            archive.seek(0)
+            self.node.run("tar -xzf -", stdin=archive)
+        self.assertEqual(seen, [b"source", b"source"])
+
+    def test_command_failure_is_not_retried(self):
+        with patch.object(gitops, "_run", side_effect=subprocess.CalledProcessError(1, "ssh")) as run:
+            with self.assertRaises(subprocess.CalledProcessError):
+                self.node.run("false")
+        self.assertEqual(run.call_count, 1)
+
+    def test_gives_up_after_retries(self):
+        with patch.object(gitops, "_run", side_effect=subprocess.CalledProcessError(255, "ssh")) as run:
+            with self.assertRaises(subprocess.CalledProcessError):
+                self.node.run("true")
+        self.assertEqual(run.call_count, gitops.BuildNode.RETRIES)
+
+    def test_close_stops_master_and_removes_socket_dir(self):
+        with patch.object(gitops, "_run") as run:
+            self.node.close()
+        self.assertIn("-O", run.call_args[0][0])
+        self.assertFalse(Path(self.node.control_dir).exists())
+
+
 def git(cwd, *args):
     return subprocess.run(["git", "-C", str(cwd), *args], check=True, capture_output=True, text=True).stdout.strip()
 
@@ -246,6 +365,7 @@ class HookTests(unittest.TestCase):
             context = self.context([("extor", host())])
             gitops._gitops_deploy(context)
         self.assertEqual(calls, ["check", "build", "secret", "pin", "sync"])
+        self.assertTrue(context["skip_remote"])
         self.assertEqual(context["gitops"], {"image": f"{IMAGE}@{NEW}", "revision": "f" * 40})
 
     def test_build_failure_publishes_nothing(self):
@@ -262,6 +382,14 @@ class HookTests(unittest.TestCase):
             gitops._gitops_deploy(self.context([("web", host(deployment_strategy="zero_downtime"))]))
         check.assert_not_called()
 
+    def test_mixed_inventory_keeps_remote_stage(self):
+        with patch.object(gitops, "check_infra_repo", return_value="main"), \
+             patch.object(gitops, "build_and_push", return_value=NEW), patch.object(gitops, "apply_env_secret"), \
+             patch.object(gitops, "publish_pin", return_value="f" * 40), patch.object(gitops, "sync_and_wait"):
+            context = self.context([("extor", host()), ("web", host(deployment_strategy="zero_downtime"))])
+            gitops._gitops_deploy(context)
+        self.assertNotIn("skip_remote", context)
+
     def test_one_host_only(self):
         with self.assertRaisesRegex(ValueError, "one host"):
             gitops._gitops_deploy(self.context([("a", host()), ("b", host())]))
@@ -269,3 +397,25 @@ class HookTests(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class RunCommandSkipTests(unittest.TestCase):
+    def test_skip_remote_runs_no_pyinfra_but_postcommand(self):
+        from djaploy import deploy
+        called = []
+
+        def fake_call_hook(name, context):
+            called.append(name)
+            if name == "deploy:precommand":
+                context["skip_remote"] = "gitops deploys from this machine"
+            return []
+
+        context = {"command": "deploy", "env": "dev", "command_file": "x", "inventory_file": "inv.py",
+                   "pyinfra_data": {}}
+        with patch("djaploy.hooks.discover_hooks"), patch("djaploy.hooks.call_hook", side_effect=fake_call_hook), \
+             patch.object(deploy, "_run_pyinfra") as pyinfra, patch.object(deploy, "_preprocess_inventory") as pre:
+            deploy.run_command(context)
+        pyinfra.assert_not_called()
+        pre.assert_not_called()
+        self.assertTrue(context["success"])
+        self.assertEqual(called, ["deploy:precommand", "precommand", "deploy:postcommand", "postcommand"])

djaploy-1.6.0/djaploy/version.py

@@ -1 +0,0 @@
-__version__ = "1.6.0"