hyperstudy-perception 0.1.0__py3-none-any.whl

perception/backends/cloud.py

@@ -0,0 +1,214 @@
+"""Cloud runs: a node on a cloud GPU machine started for it and deleted after it (`perception cloud on`).
+
+The machine is an SSH host for one attempt: the same driver (remote_driver.py) runs the node there, after first
+starting the machine through SkyPilot with this repository's project config (cloud/skypilot/sky.yaml: no laptop
+credentials on the machine, and only the perception-node role, which can end perception-* machines and nothing else).
+Before anything is sent the driver checks the machine (guard.sh: that role, no AWS credentials, and a shutdown
+deadline), and whatever happens it deletes the machine (`sky down`). Backstops if the laptop never comes back: the
+machine's shutdown deadline (an OS shutdown terminates it), and SkyPilot's autodown once idle (no jobs, no SSH).
+
+A cloud place costs money: reserved before it starts at the hourly price for the whole deadline (the most it can run),
+and charged for the time the machine existed. The deadline is 20 minutes (starting the machine, installing the node's
+environment) plus the video's length, at least 30 minutes: a GPU node slower than real time on an L4 is not expected
+(the face node ran at ~4x). Only CUDA runners are offered: the machine is a GPU machine.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import re
+import shutil
+from pathlib import Path
+
+from ..hosts import Host
+from ..manifests import Manifest, Runner
+from ..store import Artifact
+from .base import Capabilities, Estimate
+from .native import REPO
+from .remote import SshBackend
+
+SKYPILOT = REPO / "cloud" / "skypilot"
+
+
+class CloudBackend(SshBackend):
+    """One machine type (an L4 with 8 vCPUs: the face node's fastest measured setup, perception #4) in one region."""
+    task = SKYPILOT / "node-gpu.yaml"
+    project_config = SKYPILOT / "sky.yaml"
+    guard = SKYPILOT / "guard.sh"
+    setup = SKYPILOT / "node-setup.sh"
+    region = "us-east-1"
+    machine_title = "AWS g6.2xlarge, NVIDIA L4"
+    # on-demand, us-east-1, from SkyPilot's AWS catalogue on 2026-10-23 (spot had no L4 capacity there then)
+    hourly_usd = 0.9776
+    # what placement checks runners against; the driver probes the real machine (its tools come from node-gpu.yaml's setup)
+    caps = Capabilities(platform="linux/amd64", cpu_cores=8, memory_gb=32, accelerators={"cpu", "cuda"}, vram_gb=22)
+
+    def __init__(self, deadline_s: float | None = None, runner_kind: str = "native", python: str | None = None,
+                 launch_s: float = 15 * 60, down_s: float = 15 * 60, aws_s: float = 120, **paths):
+        """`deadline_s` fixes the deadline whatever the video (tests); by default it follows the video's length.
+        `launch_s`: how long starting the machine (and installing ffmpeg and uv on it) may take: within the deadline's
+        20 minutes for starting, which also cover installing the node's environment."""
+        self.launch_s, self.down_s, self.aws_s = launch_s, down_s, aws_s   # how long sky down, and an EC2 call, may take
+        host = Host("cloud", "(started for each node)", "", self.caps, 0)
+        super().__init__(host, runner_kind=runner_kind, python=python)
+        self.kind = "cloud"
+        self.title = "Cloud GPU (AWS L4)"   # short: it names the place in menus (`cloud status` gives the machine)
+        self.deadline_s = deadline_s
+        for name, value in paths.items():   # tests: stand-in task, guard
+            setattr(self, name, value)
+
+    @property
+    def machine(self) -> str | None:
+        return None   # a machine of its own for every node
+
+    def runners_for(self, manifest: Manifest) -> tuple[list[Runner], str]:
+        runners, why = super().runners_for(manifest)
+        gpu = [r for r in runners if r.accelerator == "cuda"]
+        return gpu, "" if gpu else why or "it has no CUDA runner (a cloud machine is started only for GPU work)"
+
+    def missing_tools(self, runner: Runner) -> list[str]:
+        return []   # the machine's setup installs them
+
+    def deadline(self, video_s: float | None) -> float:
+        if self.deadline_s is not None:
+            return self.deadline_s
+        if video_s is None:
+            return 60 * 60
+        # the plan (estimates.probe_seconds) and the run (decode's media.v1) read the same length, the container's
+        # duration through PyAV, so they reserve the same; whole 5 minutes keep the price a round figure
+        return max(30 * 60, math.ceil((20 * 60 + video_s) / 300) * 300)
+
+    def estimate(self, manifest: Manifest, runner: Runner, inputs: dict[str, Artifact], video_s: float | None = None) -> Estimate:
+        # the most it can cost: the machine runs at most until its deadline
+        video_s = video_s or video_seconds((a.type, a.path) for a in inputs.values())
+        return Estimate(round(self.hourly_usd * self.deadline(video_s) / 3600, 4))
+
+    def driver_spec(self, attempt_dir: Path) -> dict:
+        request = json.loads((attempt_dir / "request.json").read_text())
+        return {"host": "the cloud machine", "cloud": {
+            "cluster": cluster_name(request["run"], request["node"], request["attempt"]),
+            "task": str(self.task), "project_config": str(self.project_config), "guard": str(self.guard), "setup": str(self.setup),
+            "region": self.region, "hourly_usd": self.hourly_usd, "launch_s": self.launch_s, "down_s": self.down_s, "aws_s": self.aws_s,
+            "deadline_s": self.deadline(video_seconds((e["type"], Path(e["path"])) for e in request["inputs"].values())),
+            "sky": os.environ.get("PERCEPTION_SKY") or shutil.which("sky") or "sky",
+            "aws": os.environ.get("PERCEPTION_AWS") or shutil.which("aws") or "aws"}}
+
+    def submit(self, manifest: Manifest, runner: Runner, attempt_dir: Path, env: dict[str, str]) -> dict:
+        # SkyPilot and the AWS CLI need the user's cloud credentials, which child_env keeps from nodes: the driver is
+        # this package's code, and passes them to no node (the node runs on the machine, with sshd's environment)
+        env = {**env, **{k: v for k, v in os.environ.items() if k.startswith(("AWS_", "SKYPILOT_"))}}
+        return super().submit(manifest, runner, attempt_dir, env)
+
+
+def video_seconds(inputs) -> float | None:
+    """The video's length, from a node's inputs as (type, path): its media.v1 (decode's), else the video itself."""
+    inputs = list(inputs)
+    for type_id, path in inputs:
+        if type_id == "media.v1":
+            return float(json.loads(Path(path).read_text())["duration"])
+    for type_id, path in inputs:
+        if type_id == "video.v1":
+            from ..estimates import probe_seconds
+            return probe_seconds(Path(path))
+    return None
+
+
+def cluster_name(run_id: str, node: str, attempt: int) -> str:
+    """perception-<run>-<node>-<hash>-<attempt>: the perception-node role can end only perception-* machines, and
+    SkyPilot names are lower-case letters, digits and hyphens. The hash of the whole node id keeps two nodes whose
+    shortened names agree (face_track_a, face-track-a) on machines of their own."""
+    short = re.sub(r"[^a-z0-9-]", "-", node.lower()).strip("-")[:10] or "node"
+    return f"perception-{run_id.rsplit('-', 1)[-1]}-{short}-{hashlib.sha256(node.encode()).hexdigest()[:6]}-{attempt}"
+
+
+class RunPodBackend(CloudBackend):
+    """A RunPod pod (Secure Cloud, in the US, Canada or Europe) for each attempt: an account, credit and an API key are all it needs.
+
+    Any of several GPUs with at least 20 GB, the cheapest available first: RunPod's capacity for any one type comes and
+    goes. A pod is reached over SSH with a key made for its attempt, holds the node's inputs on its own disk only (no
+    network volume: nothing stays after it is deleted), and is deleted through the API by the driver. Its backstops if
+    this laptop never comes back: the pod removes itself at its deadline (its start command sets that before anything
+    else runs; guard.sh checks it), and the next RunPod launch deletes any perception pod past its deadline (the
+    deadline is in its name)."""
+    guard = REPO / "cloud" / "runpod" / "guard.sh"
+    # perception's own image (cloud/runpod/image): RunPod's cached PyTorch image with ffmpeg, rsync, uv and runpodctl 3
+    # added; software only. Its tag is tag.py's, from what goes into it (a test checks they agree)
+    image_tag = "e92ef7a71b30"
+    # and the digest of the build that was measured: a rebuild under the same tag (apt's versions move) is never pulled
+    image_digest = "sha256:09e6eb03f0ab44353e13b198e9f2b114b07c29af5613b33da42e2e122b7a74c4"
+    # under the repository's owner, where .github/workflows/runpod-image.yml pushes it (a test checks)
+    image = f"ghcr.io/hyperstudyio/perception-runpod:{image_tag}@{image_digest}"
+    start = "/perception-start.sh"   # its own start (host keys, PUBLIC_KEY, sshd), run after the pod's removal is set
+    # Secure Cloud prices on 2026-10-24 (RunPod's gpuTypes securePrice), the order RunPod tries them in: the cheap ones
+    # first, the dearer ones when none of those is free (capacity for any one type comes and goes). Ampere, Ada and
+    # Blackwell (the nodes' torch is built for CUDA 13, which has Blackwell's kernels); not MIG slices of one.
+    gpus = {"NVIDIA RTX A4500": 0.25, "NVIDIA RTX A5000": 0.27, "NVIDIA RTX 4000 Ada Generation": 0.28,
+            "NVIDIA GeForce RTX 3090": 0.50, "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition": 0.50,
+            "NVIDIA RTX PRO 4000 Blackwell": 0.57, "NVIDIA A40": 0.59, "NVIDIA L4": 0.59, "NVIDIA RTX A6000": 0.59,
+            "NVIDIA RTX PRO 4500 Blackwell": 0.72, "NVIDIA L40": 0.82,
+            "NVIDIA GeForce RTX 4090": 0.89, "NVIDIA RTX PRO 5000 Blackwell": 0.96, "NVIDIA RTX 6000 Ada Generation": 0.99,
+            "NVIDIA L40S": 1.09, "NVIDIA GeForce RTX 5090": 1.19}
+    # datacenters in the US, Canada and Europe, whichever has a GPU free: those RunPod's REST API accepts (its schema's
+    # enum, 2026-10-25; gpus above are in its enum too)
+    datacenters = ["US-CA-2", "US-DE-1", "US-GA-1", "US-GA-2", "US-IL-1", "US-KS-2", "US-KS-3", "US-MD-1", "US-MO-1", "US-MO-2",
+                   "US-NC-1", "US-TX-1", "US-TX-3", "US-TX-4", "US-WA-1", "CA-MTL-1", "CA-MTL-2", "CA-MTL-3",
+                   "EU-CZ-1", "EU-FR-1", "EU-NL-1", "EU-RO-1", "EU-SE-1", "EUR-IS-1", "EUR-IS-2", "EUR-IS-3", "EUR-NO-1"]
+    disk_gb = 40   # the container's disk: the node's environment, weights and inputs
+    # the nodes' torch is PyPI's CUDA 13 build (envs/faces/uv.lock): an NVIDIA driver of 580 or later (guard.sh)
+    min_driver = 580
+    region = "US, Canada, Europe (Secure Cloud)"
+    machine_title = "RunPod, one of " + ", ".join(g.removeprefix("NVIDIA ").removeprefix("GeForce ") for g in gpus)
+    hourly_usd = max(gpus.values())   # reserved at the dearest one it may get; charged at the price of the one it got
+    caps = Capabilities(platform="linux/amd64", cpu_cores=8, memory_gb=30, accelerators={"cpu", "cuda"}, vram_gb=20)
+
+    def __init__(self, api_key: str | None = None, api_s: float = 60, unknown_s: float = 180, **kw):
+        """`api_s`: how long one API call may take. `unknown_s`: how long a pod whose create got no clear answer is
+        looked for by its name before it is taken as never made (within down_s)."""
+        super().__init__(**kw)
+        self.api_s, self.unknown_s = api_s, unknown_s
+        self.kind = "runpod"
+        self.title = "Cloud GPU (RunPod)"
+        self.api_key = api_key
+
+    def driver_spec(self, attempt_dir: Path) -> dict:
+        from ..runpod import api_base
+        request = json.loads((attempt_dir / "request.json").read_text())
+        return {"host": "the RunPod pod", "cloud": {
+            "provider": "runpod", "name": cluster_name(request["run"], request["node"], request["attempt"]),
+            "api": api_base(), "image": self.image, "start": self.start, "gpus": list(self.gpus), "datacenters": self.datacenters,
+            "disk_gb": self.disk_gb, "min_driver": self.min_driver, "guard": str(self.guard), "setup": str(self.setup), "hourly_usd": self.hourly_usd,
+            "launch_s": self.launch_s, "down_s": self.down_s, "api_s": self.api_s, "unknown_s": self.unknown_s,
+            "deadline_s": self.deadline(video_seconds((e["type"], Path(e["path"])) for e in request["inputs"].values()))}}
+
+    def submit(self, manifest: Manifest, runner: Runner, attempt_dir: Path, env: dict[str, str]) -> dict:
+        # the account key for the driver alone (this package's code), in its environment, never in remote.json: it
+        # creates and deletes the pod from here, and sends the key nowhere
+        return SshBackend.submit(self, manifest, runner, attempt_dir, {**env, "RUNPOD_API_KEY": self.api_key or ""})
+
+
+class Cloud:
+    """cloud.json in the store's home: which cloud places are offered (`perception cloud on [aws|runpod]`)."""
+    providers = ("aws", "runpod")
+
+    def __init__(self, home: Path):
+        self.path = home / "cloud.json"
+
+    def read(self) -> dict:
+        return json.loads(self.path.read_text()) if self.path.exists() else {}
+
+    @property
+    def enabled(self) -> bool:   # AWS (the first cloud place: its key is "enabled")
+        return self.read().get("enabled", False)
+
+    @property
+    def runpod(self) -> bool:
+        return self.read().get("runpod", False)
+
+    def set(self, enabled: bool, provider: str = "aws"):
+        doc = self.read()
+        doc["enabled" if provider == "aws" else provider] = enabled
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps(doc))

perception/backends/native.py

@@ -0,0 +1,140 @@
+"""Native runs: the node's script in its uv environment on this machine (kind `native`), or in the coordinator's own
+interpreter (kind `self`, for tests and trivial nodes). This is the backend that reaches the Apple GPU: containers on
+macOS cannot.
+
+Each node runs in its own process group with its output going to the attempt's log.txt. The handle records the pid
+and the request path, so a coordinator that restarts can tell whether an attempt is still running (and wait for it)
+instead of starting a second copy.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import signal
+import subprocess
+import sys
+from pathlib import Path
+
+from ..manifests import Manifest, Runner
+from ..store import Artifact
+from .base import Backend, Capabilities
+from .probe import probe
+
+from ..release import REPO, env_vars
+
+
+def sdk_hash() -> str:
+    h = hashlib.sha256()
+    for p in sorted((REPO / "sdk" / "src").rglob("*.py")):
+        h.update(p.read_bytes())
+    return h.hexdigest()
+
+
+class NativeBackend(Backend):
+    kind = "native"
+
+    def __init__(self):
+        self._children: dict[int, subprocess.Popen] = {}
+        self._caps: Capabilities | None = None
+
+    def probe_capabilities(self) -> Capabilities:
+        if self._caps is None:
+            self._caps = probe()
+        return self._caps
+
+    def missing_tools(self, runner: Runner) -> list[str]:
+        return [] if self.probe_capabilities().tools.get("uv") else ["uv"]
+
+    def identity(self, manifest: Manifest, runner: Runner) -> str:
+        lock = REPO / runner.env / "uv.lock" if runner.env else None
+        h = hashlib.sha256(manifest.code_hash().encode())
+        h.update(sdk_hash().encode())   # the node SDK shapes what a node writes; it is installed editable, outside the lock
+        h.update(lock.read_bytes() if lock and lock.exists() else b"no lock")
+        h.update(runner.accelerator.encode())
+        return f"native:{h.hexdigest()[:16]}"
+
+    def stage_inputs(self, attempt_dir: Path, inputs: dict[str, Artifact]) -> dict[str, dict]:
+        # same machine: the node reads published objects in place (they are read-only)
+        return {name: {"type": a.type, "sha256": a.hash, "path": str(a.path)} for name, a in inputs.items()}
+
+    def command(self, manifest: Manifest, runner: Runner, request: Path) -> list[str]:
+        entry = str(manifest.directory / runner.entry)
+        if self.kind == "self":
+            return [sys.executable, entry, "--request", str(request)]
+        if not runner.env:
+            raise ValueError(f"{manifest.ref}: a native runner needs env (a uv project)")
+        return [shutil.which("uv") or "uv", "run", "--project", str(REPO / runner.env), "--frozen", "--no-dev",
+                "python", entry, "--request", str(request)]
+
+    def submit(self, manifest: Manifest, runner: Runner, attempt_dir: Path, env: dict[str, str]) -> dict:
+        request = attempt_dir / "request.json"
+        log = open(attempt_dir / "log.txt", "ab")
+        env = {**env, **(env_vars(runner.env) if self.kind == "native" and runner.env else {})}   # the package's: in the store
+        proc = subprocess.Popen(self.command(manifest, runner, request), cwd=manifest.directory, env=env,
+                                stdout=log, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL,
+                                start_new_session=True)   # its own process group: cancel stops the whole tree
+        log.close()
+        self._children[proc.pid] = proc
+        handle = {"pid": proc.pid, "request": str(request)}
+        tmp = attempt_dir / "handle.json.tmp"   # at once, before the coordinator records it: see recorded_handle
+        tmp.write_text(json.dumps(handle))
+        os.replace(tmp, attempt_dir / "handle.json")
+        return handle
+
+    def recorded_handle(self, attempt_dir: Path) -> dict | None:
+        """Trusted only as a handle for this attempt: the node can write in its folder, so a handle naming another request,
+        or a pid that is not one, steers no kill or adoption."""
+        path = attempt_dir / "handle.json"
+        if not path.is_file() or path.is_symlink():
+            return None
+        try:
+            h = json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            return None
+        # a pid in the range any OS gives (Linux's PID_MAX_LIMIT is 2^22): larger ones would overflow os.kill
+        ok = (isinstance(h, dict) and set(h) == {"pid", "request"} and type(h["pid"]) is int and 1 < h["pid"] <= 2 ** 22
+              and h["request"] == str(attempt_dir / "request.json"))
+        return h if ok else None
+
+    def poll(self, handle: dict) -> str:
+        pid = handle["pid"]
+        proc = self._children.get(pid)
+        if proc is not None:   # our own child: poll() also reaps it
+            return "running" if proc.poll() is None else "exited"
+        return "running" if _alive(pid, handle["request"]) else "exited"   # an attempt a previous coordinator started
+
+    def cancel(self, handle: dict) -> None:
+        pid = handle["pid"]
+        if self.poll(handle) == "running":
+            try:
+                os.killpg(pid, signal.SIGTERM)
+            except (ProcessLookupError, PermissionError):
+                pass   # already gone, or exiting: macOS reports EPERM for a group whose leader is a zombie
+
+
+class SelfBackend(NativeBackend):
+    kind = "self"
+
+    def missing_tools(self, runner: Runner) -> list[str]:
+        return []
+
+    def identity(self, manifest: Manifest, runner: Runner) -> str:
+        # the accelerator too: a place is a choice, and another place's result must not be served from the cache
+        return f"self:{hashlib.sha256((manifest.code_hash() + sdk_hash() + runner.accelerator).encode()).hexdigest()[:16]}"
+
+
+def _alive(pid: int, request: str) -> bool:
+    """Is `pid` still the node process for this request? (pids are reused, so check its command line too.)"""
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return False
+    # -ww: unlimited width (procps truncates columns to 80 characters when not writing to a terminal, and a long request
+    # path then made a live node look dead)
+    out = subprocess.run(["ps", "-ww", "-p", str(pid), "-o", "command="], capture_output=True, text=True)
+    return request in out.stdout

perception/backends/probe.py

@@ -0,0 +1,44 @@
+"""What this machine has: platform, cores, memory, accelerators, tools. Used by `perception doctor` and placement."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+
+from .base import Capabilities, this_platform
+
+
+def _memory_gb() -> float:
+    if sys.platform == "darwin":
+        out = subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True, text=True)
+        return int(out.stdout.strip()) / 2**30 if out.returncode == 0 else 0.0
+    if hasattr(os, "sysconf") and "SC_PHYS_PAGES" in os.sysconf_names:
+        return os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") / 2**30
+    return 0.0
+
+
+def _cuda_vram_gb() -> float:
+    smi = shutil.which("nvidia-smi")
+    if not smi:
+        return 0.0
+    out = subprocess.run([smi, "--query-gpu=memory.total", "--format=csv,noheader,nounits"], capture_output=True, text=True)
+    if out.returncode:
+        return 0.0
+    return max((float(x) / 1024 for x in out.stdout.split() if x.strip()), default=0.0)
+
+
+def probe() -> Capabilities:
+    plat = this_platform()
+    memory = _memory_gb()
+    accelerators = {"cpu"}
+    vram = _cuda_vram_gb()
+    if vram:
+        accelerators.add("cuda")
+    if plat == "darwin/arm64":
+        accelerators.add("mps")
+        vram = vram or memory   # unified memory
+    return Capabilities(platform=plat, cpu_cores=os.cpu_count() or 1, memory_gb=round(memory, 1), accelerators=accelerators,
+                        vram_gb=round(vram, 1), tools={"uv": shutil.which("uv"), "ffmpeg": shutil.which("ffmpeg"),
+                                                       "ffprobe": shutil.which("ffprobe")})

perception/backends/remote.py

@@ -0,0 +1,140 @@
+"""Remote runs: a node on another machine, reached over SSH (a lab server, `perception hosts add`).
+
+The node runs there exactly as a native node runs here (its script in its uv environment, the same file protocol), but
+the coordinator never talks to the machine itself. `submit` starts a local driver process (`python -m
+perception.remote`, remote_driver.py) that copies the node's code and inputs over, starts it, streams its log back,
+and brings out/ and response.json back into the attempt's folder. To the coordinator the driver is a local process
+like any native node: poll, cancel (the driver stops the remote node), adoption after a coordinator restart and
+collect all work unchanged.
+
+What reaches the other machine: the files git tracks for the node, its uv project and the node SDK, and the node's
+inputs. Those inputs leave this machine, so a remote place is offered only when the graph's policy allows each kind of
+data in them (policy.allow_upload), and runs only with the run's consent (--allow-uploads), as a cloud API's node does.
+Nodes that send data out themselves (to an API) or need secrets are not run remotely: their data would go to two
+places, and secrets stay here.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+from ..hosts import Host, ssh_command
+from ..manifests import Manifest, Runner
+from ..release import files_in, release
+from .base import Capabilities
+from .native import REPO, NativeBackend
+
+
+def code_files(manifest: Manifest, runner: Runner) -> list[str]:
+    """What a node needs on the other machine, relative to the repository: the files git tracks in its directory, its
+    uv project and the node SDK. Only tracked files: a working copy can hold images, crops and bundles that must not
+    leave the machine (as launch.sh stages for whole-graph cloud runs)."""
+    if not manifest.directory.resolve().is_relative_to(REPO):
+        raise RuntimeError(f"{manifest.ref} is not in the pipeline's repository ({manifest.directory}): only files it tracks are sent")
+    dirs = [manifest.directory.resolve().relative_to(REPO).as_posix(), "sdk"]
+    if runner.env:
+        dirs.append(runner.env)
+    entry_dir = (manifest.directory / runner.entry).resolve().parent.relative_to(REPO).as_posix()
+    if not (entry_dir + "/").startswith(dirs[0] + "/"):
+        dirs.append(entry_dir)   # a script shared with another node (../work/node.py)
+    if release():
+        files = files_in(dirs)   # a copy shipped with an app: every file in it was tracked where it was made
+    else:
+        out = subprocess.run(["git", "-C", str(REPO), "ls-files", "-z", "--", *dirs], capture_output=True, text=True)
+        if out.returncode:
+            raise RuntimeError(f"{REPO} is not a git checkout: only files git tracks are sent to other machines ({out.stderr.strip()})")
+        files = sorted(f for f in out.stdout.split("\0") if f)
+    refuse_links(files)
+    if not any(f.startswith(dirs[0] + "/") for f in files):
+        raise RuntimeError(f"git tracks no files in {dirs[0]}: commit the node before running it on another machine")
+    return files
+
+
+def refuse_links(files: list[str], root: Path = REPO):
+    """A link git tracks would be followed when the code is copied and hashed, and what it points at (anything on this
+    machine) would go to the host: only content git tracks leaves the machine, so a node with one is not sent."""
+    links = [f for f in files if (root / f).is_symlink()]
+    if links:
+        raise RuntimeError(f"git tracks links among the code to send ({', '.join(links[:5])}): links are not sent to other machines")
+
+
+def code_hash(files: list[str], root: Path = REPO) -> str:
+    """The code's content, as the name of its folder on the other machine: the same code is sent once."""
+    h = hashlib.sha256()
+    for f in files:
+        h.update(f.encode() + b"\0" + hashlib.sha256((root / f).read_bytes()).digest())
+    return h.hexdigest()[:20]
+
+
+class SshBackend(NativeBackend):
+    runner_kind = "native"
+    remote = True
+
+    def __init__(self, host: Host, runner_kind: str = "native", python: str | None = None, lost_after_s: float = 600):
+        """`runner_kind="self"` with `python` (an interpreter on the other machine that has the node SDK) is for tests."""
+        super().__init__()
+        self.host = host
+        self.kind = f"ssh-{host.name}"
+        self.title = f"{host.name} (SSH)"
+        self.runner_kind = runner_kind
+        self.python = python
+        self.lost_after_s = lost_after_s
+
+    @property
+    def machine(self) -> str | None:
+        return self.kind
+
+    def probe_capabilities(self) -> Capabilities:
+        return self.host.caps   # as recorded by `hosts add` / `hosts check`: planning never waits on the network
+
+    def missing_tools(self, runner: Runner) -> list[str]:
+        if self.runner_kind == "self":
+            return []
+        return [] if self.host.caps.tools.get("uv") else [f"uv (on {self.host.name})"]
+
+    def remote_refusal(self, manifest: Manifest) -> str:
+        if manifest.privacy.external_upload:
+            return f"it sends {', '.join(manifest.privacy.external_upload)} to {manifest.service or 'a service'} itself, so it runs on this machine"
+        if manifest.secrets:
+            return f"it needs {', '.join(manifest.secrets)}, and secrets are not sent to other machines"
+        if not manifest.directory.resolve().is_relative_to(REPO):
+            return "it is not in the pipeline's repository, and only files the repository tracks are sent to other machines"
+        return ""
+
+    def identity(self, manifest: Manifest, runner: Runner) -> str:
+        """Exactly the code that runs there (what code_files sends: the node's tracked files, a shared entry script, its
+        uv project with its lock, the SDK), the accelerator and the host's platform."""
+        try:
+            code = code_hash(code_files(manifest, runner))
+        except (RuntimeError, OSError) as e:   # nothing could be sent (not a git checkout, a tracked file deleted): submit says so; no cache entry is shared
+            code = f"unsendable {e}"
+        h = hashlib.sha256(f"{code} {runner.accelerator} {self.host.caps.platform}".encode())
+        return f"{self.kind}:{h.hexdigest()[:16]}"
+
+    def driver_spec(self, attempt_dir: Path) -> dict:
+        """What else the driver needs (a cloud machine to start first)."""
+        return {}
+
+    def command(self, manifest: Manifest, runner: Runner, request: Path) -> list[str]:
+        # the request path on the command line: how a restarted coordinator recognises the driver (native._alive)
+        return [sys.executable, "-m", "perception.remote_driver", "--request", str(request)]
+
+    def submit(self, manifest: Manifest, runner: Runner, attempt_dir: Path, env: dict[str, str]) -> dict:
+        files = code_files(manifest, runner)
+        caps = self.host.caps
+        spec = {
+            "dest": self.host.dest, "root": self.host.root, "host": self.host.name, "ssh": ssh_command(),
+            "code": code_hash(files), "files": files,
+            "node_dir": manifest.directory.resolve().relative_to(REPO).as_posix(), "entry": runner.entry, "env": runner.env,
+            "runner_kind": self.runner_kind, "python": self.python, "uv": caps.tools.get("uv"), "prepare": manifest.prepare,
+            # the folders of the tools it found: ssh commands get a minimal PATH (ffmpeg in /opt/homebrew/bin on a Mac)
+            "path": sorted({str(Path(p).parent) for p in caps.tools.values() if p}),
+            "lost_after_s": self.lost_after_s,
+            **self.driver_spec(attempt_dir),
+        }
+        (attempt_dir / "remote.json").write_text(json.dumps(spec, indent=1))
+        return super().submit(manifest, runner, attempt_dir, env)