msdev 0.9.0__py3-none-any.whl

msdev/core/transport.py

@@ -0,0 +1,1225 @@
+"""Local and OpenSSH-backed RPC transports."""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import queue
+import shlex
+import signal
+import socket
+import subprocess
+import sys
+import tarfile
+import tempfile
+import threading
+import time
+import uuid
+from pathlib import Path
+from typing import Any, Callable, Protocol
+
+from .config import (
+    default_socket_path,
+    parse_environment_spec,
+    wrap_conda_command,
+)
+from .limits import (
+    DEFAULT_EXEC_TIMEOUT_SECONDS,
+    EXEC_RPC_COMPLETION_GRACE_SECONDS,
+    MAX_RPC_ERROR_BYTES,
+    MAX_RPC_RESPONSE_BYTES,
+    RPC_PROCESS_CLEANUP_GRACE_SECONDS,
+    validate_exec_timeout_seconds,
+)
+from .resources import Environment, Node, parse_variable_assignment
+
+
+SCP_BOOTSTRAP_TIMEOUT_SECONDS = 120
+REMOTE_CLEANUP_TIMEOUT_SECONDS = 10
+LOCAL_RPC_TIMEOUT_SECONDS = 60.0
+SSH_RPC_TIMEOUT_SECONDS = 120.0
+
+
+def _rpc_timeout_seconds(
+    method: str,
+    params: dict[str, Any] | None,
+    default: float,
+) -> float | None:
+    if method not in {"exec", "exec.stream"}:
+        return default
+    timeout = (params or {}).get(
+        "timeout_seconds",
+        DEFAULT_EXEC_TIMEOUT_SECONDS,
+    )
+    timeout = validate_exec_timeout_seconds(timeout)
+    if timeout == -1:
+        return None
+    return timeout + EXEC_RPC_COMPLETION_GRACE_SECONDS
+
+
+class RpcError(RuntimeError):
+    pass
+
+
+class DaemonUnavailableError(RpcError):
+    pass
+
+
+class _BoundedOutputError(RuntimeError):
+    def __init__(self, stream_name: str):
+        self.stream_name = stream_name
+        super().__init__(stream_name)
+
+
+def _run_bounded_process(
+    args: list[str],
+    *,
+    input_data: bytes,
+    timeout: float | None,
+    stdout_limit: int,
+    stderr_limit: int,
+    keep_stdin_open: bool = False,
+) -> subprocess.CompletedProcess[bytes]:
+    process = subprocess.Popen(
+        args,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        start_new_session=(os.name == "posix"),
+    )
+    assert process.stdin is not None
+    assert process.stdout is not None
+    assert process.stderr is not None
+    stdout = bytearray()
+    stderr = bytearray()
+    overflow: list[str] = []
+    overflow_lock = threading.Lock()
+
+    def terminate_process_group() -> None:
+        if process.poll() is not None:
+            return
+        try:
+            if os.name == "posix":
+                os.killpg(process.pid, signal.SIGKILL)
+            else:
+                process.kill()
+        except (OSError, ProcessLookupError):
+            pass
+
+    def drain(stream: Any, retained: bytearray, limit: int, name: str) -> None:
+        try:
+            while True:
+                chunk = stream.read(64 * 1024)
+                if not chunk:
+                    return
+                if len(retained) + len(chunk) > limit:
+                    with overflow_lock:
+                        if not overflow:
+                            overflow.append(name)
+                    terminate_process_group()
+                    return
+                retained.extend(chunk)
+        except (OSError, ValueError):
+            return
+
+    def write_input() -> None:
+        try:
+            process.stdin.write(input_data)
+            process.stdin.flush()
+        except (BrokenPipeError, OSError):
+            pass
+        finally:
+            if not keep_stdin_open:
+                try:
+                    process.stdin.close()
+                except OSError:
+                    pass
+
+    threads = [
+        threading.Thread(
+            target=drain,
+            args=(process.stdout, stdout, stdout_limit, "stdout"),
+            daemon=True,
+        ),
+        threading.Thread(
+            target=drain,
+            args=(process.stderr, stderr, stderr_limit, "stderr"),
+            daemon=True,
+        ),
+        threading.Thread(target=write_input, daemon=True),
+    ]
+    for thread in threads:
+        thread.start()
+    timed_out: subprocess.TimeoutExpired | None = None
+    interrupted: BaseException | None = None
+    try:
+        returncode = process.wait(timeout=timeout)
+    except subprocess.TimeoutExpired as exc:
+        timed_out = exc
+        terminate_process_group()
+        returncode = 124
+    except BaseException as exc:
+        interrupted = exc
+        terminate_process_group()
+        returncode = 130
+    cleanup_deadline = time.monotonic() + RPC_PROCESS_CLEANUP_GRACE_SECONDS
+    try:
+        if process.poll() is None:
+            try:
+                process.wait(
+                    timeout=max(0.0, cleanup_deadline - time.monotonic())
+                )
+            except subprocess.TimeoutExpired:
+                terminate_process_group()
+        for thread in threads:
+            thread.join(max(0.0, cleanup_deadline - time.monotonic()))
+        for stream in (process.stdin, process.stdout, process.stderr):
+            try:
+                stream.close()
+            except OSError:
+                pass
+        for thread in threads:
+            if thread.is_alive():
+                thread.join(max(0.0, cleanup_deadline - time.monotonic()))
+    finally:
+        terminate_process_group()
+
+    if timed_out is not None:
+        raise timed_out
+    if interrupted is not None:
+        raise interrupted
+
+    if overflow:
+        raise _BoundedOutputError(overflow[0])
+    return subprocess.CompletedProcess(
+        args=args,
+        returncode=returncode,
+        stdout=bytes(stdout),
+        stderr=bytes(stderr),
+    )
+
+
+def _run_streaming_process(
+    args: list[str],
+    *,
+    input_data: bytes,
+    timeout: float | None,
+    stdout_limit: int,
+    stderr_limit: int,
+    on_line: Callable[[bytes], None],
+) -> subprocess.CompletedProcess[bytes]:
+    process = subprocess.Popen(
+        args,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        start_new_session=(os.name == "posix"),
+    )
+    assert process.stdin is not None
+    assert process.stdout is not None
+    assert process.stderr is not None
+    lines: queue.Queue[bytes | None] = queue.Queue(maxsize=16)
+    stderr = bytearray()
+    overflow: list[str] = []
+    stop = threading.Event()
+
+    def terminate_process_group() -> None:
+        if process.poll() is not None:
+            return
+        try:
+            if os.name == "posix":
+                os.killpg(process.pid, signal.SIGKILL)
+            else:
+                process.kill()
+        except (OSError, ProcessLookupError):
+            pass
+
+    def enqueue(value: bytes | None) -> None:
+        while not stop.is_set():
+            try:
+                lines.put(value, timeout=0.1)
+                return
+            except queue.Full:
+                continue
+
+    def read_stdout() -> None:
+        try:
+            while not stop.is_set():
+                line = process.stdout.readline(stdout_limit + 1)
+                if not line:
+                    break
+                if len(line) > stdout_limit:
+                    overflow.append("stdout")
+                    terminate_process_group()
+                    break
+                enqueue(line)
+        except (OSError, ValueError):
+            pass
+        finally:
+            enqueue(None)
+
+    def read_stderr() -> None:
+        try:
+            while not stop.is_set():
+                chunk = process.stderr.read(64 * 1024)
+                if not chunk:
+                    return
+                if len(stderr) + len(chunk) > stderr_limit:
+                    overflow.append("stderr")
+                    terminate_process_group()
+                    return
+                stderr.extend(chunk)
+        except (OSError, ValueError):
+            return
+
+    def write_input() -> None:
+        try:
+            process.stdin.write(input_data)
+            process.stdin.flush()
+        except (BrokenPipeError, OSError):
+            pass
+
+    threads = [
+        threading.Thread(target=read_stdout, daemon=True),
+        threading.Thread(target=read_stderr, daemon=True),
+        threading.Thread(target=write_input, daemon=True),
+    ]
+    for thread in threads:
+        thread.start()
+    deadline = None if timeout is None else time.monotonic() + timeout
+    try:
+        while True:
+            if overflow:
+                raise _BoundedOutputError(overflow[0])
+            remaining = (
+                None if deadline is None else deadline - time.monotonic()
+            )
+            if remaining is not None and remaining <= 0:
+                raise subprocess.TimeoutExpired(args, timeout)
+            try:
+                line = lines.get(
+                    timeout=0.1
+                    if remaining is None
+                    else min(0.1, remaining)
+                )
+            except queue.Empty:
+                continue
+            if line is None:
+                break
+            on_line(line)
+        remaining = (
+            None
+            if deadline is None
+            else max(0.0, deadline - time.monotonic())
+        )
+        returncode = process.wait(timeout=remaining)
+    except BaseException:
+        terminate_process_group()
+        try:
+            process.wait(timeout=RPC_PROCESS_CLEANUP_GRACE_SECONDS)
+        except subprocess.TimeoutExpired:
+            terminate_process_group()
+        raise
+    finally:
+        stop.set()
+        for stream in (process.stdin, process.stdout, process.stderr):
+            try:
+                stream.close()
+            except OSError:
+                pass
+        cleanup_deadline = (
+            time.monotonic() + RPC_PROCESS_CLEANUP_GRACE_SECONDS
+        )
+        for thread in threads:
+            thread.join(max(0.0, cleanup_deadline - time.monotonic()))
+
+    return subprocess.CompletedProcess(
+        args=args,
+        returncode=returncode,
+        stdout=b"",
+        stderr=bytes(stderr),
+    )
+
+
+class RpcTransport(Protocol):
+    def call(self, method: str, params: dict[str, Any] | None = None) -> Any: ...
+
+    def stream_exec(
+        self,
+        params: dict[str, Any],
+        on_output: Callable[[str, bytes], None],
+    ) -> dict[str, Any]: ...
+
+
+def _request(
+    method: str,
+    params: dict[str, Any] | None = None,
+    *,
+    cancel_on_disconnect: bool = False,
+) -> tuple[str, bytes]:
+    request_id = uuid.uuid4().hex
+    payload = {
+        "id": request_id,
+        "method": method,
+        "params": params or {},
+    }
+    if cancel_on_disconnect:
+        payload["cancel_on_disconnect"] = True
+    return request_id, json.dumps(payload, ensure_ascii=False).encode("utf-8") + b"\n"
+
+
+def _bounded_utf8(value: Any, maximum: int) -> str:
+    text = str(value)
+    retained = bytearray()
+    for offset in range(0, len(text), 4096):
+        chunk = text[offset : offset + 4096].encode("utf-8", errors="replace")
+        remaining = maximum - len(retained)
+        if remaining <= 0:
+            break
+        retained.extend(chunk[:remaining])
+    return bytes(retained).decode("utf-8", errors="ignore")
+
+
+def _decode_response(request_id: str, raw: bytes) -> Any:
+    try:
+        response = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise RpcError(f"invalid RPC response: {raw[:500]!r}") from exc
+    if response.get("id") not in {request_id, None}:
+        raise RpcError("RPC response id mismatch")
+    if "error" in response:
+        error = response["error"]
+        if response.get("id") is None and error.get("type") in {
+            "FileNotFoundError",
+            "ConnectionRefusedError",
+        }:
+            raise DaemonUnavailableError(
+                "remote msdevd is not running or its socket is unavailable; "
+                "run `msdev node bootstrap <node>`"
+            )
+        error_type = _bounded_utf8(error.get("type", "Error"), 128)
+        message = _bounded_utf8(error.get("message", ""), MAX_RPC_ERROR_BYTES)
+        raise RpcError(f"{error_type}: {message}")
+    return response.get("result")
+
+
+def _decode_stream_event(
+    request_id: str,
+    raw: bytes,
+    on_output: Callable[[str, bytes], None],
+) -> dict[str, Any] | None:
+    try:
+        event = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise RpcError(f"invalid streaming RPC event: {raw[:500]!r}") from exc
+    if not isinstance(event, dict):
+        raise RpcError("streaming RPC event must be an object")
+    if "error" in event and "event" not in event:
+        _decode_response(request_id, raw)
+        raise RpcError("streaming RPC returned an invalid error response")
+    if event.get("id") != request_id:
+        raise RpcError("streaming RPC response id mismatch")
+    event_type = event.get("event")
+    if event_type in {"stdout", "stderr"}:
+        encoded = event.get("data_base64")
+        if not isinstance(encoded, str):
+            raise RpcError("streaming RPC output event has invalid data")
+        try:
+            chunk = base64.b64decode(encoded, validate=True)
+        except (ValueError, TypeError) as exc:
+            raise RpcError("streaming RPC output event has invalid base64") from exc
+        on_output(event_type, chunk)
+        return None
+    if event_type == "result":
+        result = event.get("result")
+        if not isinstance(result, dict):
+            raise RpcError("streaming RPC result event has invalid result")
+        return result
+    if event_type == "error":
+        error = event.get("error") or {}
+        error_type = _bounded_utf8(error.get("type", "Error"), 128)
+        message = _bounded_utf8(error.get("message", ""), MAX_RPC_ERROR_BYTES)
+        raise RpcError(f"{error_type}: {message}")
+    raise RpcError(f"unsupported streaming RPC event: {event_type!r}")
+
+
+class UnixRpcTransport:
+    def __init__(self, socket_path: Path | None = None):
+        self.socket_path = socket_path or default_socket_path()
+
+    def call(self, method: str, params: dict[str, Any] | None = None) -> Any:
+        request_id, payload = _request(method, params)
+        rpc_timeout = _rpc_timeout_seconds(
+            method,
+            params,
+            LOCAL_RPC_TIMEOUT_SECONDS,
+        )
+        deadline = (
+            None if rpc_timeout is None else time.monotonic() + rpc_timeout
+        )
+        try:
+            with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
+                client.settimeout(
+                    None
+                    if deadline is None
+                    else max(0.001, deadline - time.monotonic())
+                )
+                client.connect(str(self.socket_path.expanduser()))
+                client.settimeout(
+                    None
+                    if deadline is None
+                    else max(0.001, deadline - time.monotonic())
+                )
+                client.sendall(payload)
+                response = bytearray()
+                while not response.endswith(b"\n"):
+                    if deadline is not None:
+                        remaining = deadline - time.monotonic()
+                        if remaining <= 0:
+                            raise TimeoutError("RPC deadline exceeded")
+                        client.settimeout(remaining)
+                    chunk = client.recv(65536)
+                    if not chunk:
+                        break
+                    if len(response) + len(chunk) > MAX_RPC_RESPONSE_BYTES:
+                        raise RpcError(
+                            "RPC response exceeded the configured byte limit"
+                        )
+                    response.extend(chunk)
+        except OSError as exc:
+            detail = _bounded_utf8(exc, MAX_RPC_ERROR_BYTES)
+            raise RpcError(
+                f"cannot reach local msdevd at {self.socket_path}: {detail}"
+            ) from exc
+        return _decode_response(request_id, bytes(response))
+
+    def stream_exec(
+        self,
+        params: dict[str, Any],
+        on_output: Callable[[str, bytes], None],
+    ) -> dict[str, Any]:
+        request_id, payload = _request("exec.stream", params)
+        rpc_timeout = _rpc_timeout_seconds(
+            "exec.stream",
+            params,
+            LOCAL_RPC_TIMEOUT_SECONDS,
+        )
+        deadline = (
+            None if rpc_timeout is None else time.monotonic() + rpc_timeout
+        )
+        buffer = bytearray()
+        try:
+            with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
+                client.settimeout(
+                    None
+                    if deadline is None
+                    else max(0.001, deadline - time.monotonic())
+                )
+                client.connect(str(self.socket_path.expanduser()))
+                client.sendall(payload)
+                while True:
+                    if deadline is not None:
+                        remaining = deadline - time.monotonic()
+                        if remaining <= 0:
+                            raise TimeoutError("RPC deadline exceeded")
+                        client.settimeout(remaining)
+                    chunk = client.recv(65536)
+                    if not chunk:
+                        break
+                    buffer.extend(chunk)
+                    while b"\n" in buffer:
+                        raw, _, remainder = buffer.partition(b"\n")
+                        buffer = bytearray(remainder)
+                        result = _decode_stream_event(
+                            request_id,
+                            bytes(raw),
+                            on_output,
+                        )
+                        if result is not None:
+                            return result
+                    if len(buffer) > MAX_RPC_RESPONSE_BYTES:
+                        raise RpcError(
+                            "streaming RPC event exceeded the configured byte limit"
+                        )
+        except OSError as exc:
+            detail = _bounded_utf8(exc, MAX_RPC_ERROR_BYTES)
+            raise RpcError(
+                f"cannot reach local msdevd at {self.socket_path}: {detail}"
+            ) from exc
+        raise RpcError("streaming RPC ended before the result event")
+
+
+class SshRpcTransport:
+    def __init__(
+        self,
+        node: Node,
+        environment: Environment | None = None,
+        *,
+        auto_bootstrap: bool | None = None,
+    ):
+        self.node = node
+        self.environment = environment or Environment(node.name, node.name)
+        self.auto_bootstrap = (
+            node.auto_bootstrap if auto_bootstrap is None else auto_bootstrap
+        )
+        control_dir = Path(os.environ.get("MSDEV_SSH_CONTROL_DIR", "~/.cache/msdev/ssh")).expanduser()
+        control_dir.mkdir(parents=True, exist_ok=True)
+        self.control_path = str(control_dir / "%C")
+        self.control_persist = os.environ.get("MSDEV_SSH_CONTROL_PERSIST", "yes")
+
+    def _node_environment(self) -> dict[str, str]:
+        return dict(
+            parse_variable_assignment(value)
+            for value in self.node.variables
+        )
+
+    def _ssh_options(self) -> list[str]:
+        return [
+            "-o",
+            "ControlMaster=auto",
+            "-o",
+            f"ControlPersist={self.control_persist}",
+            "-o",
+            f"ControlPath={self.control_path}",
+            "-o",
+            "ConnectTimeout=10",
+            "-o",
+            "ServerAliveInterval=15",
+            "-o",
+            "ServerAliveCountMax=2",
+        ]
+
+    def _ssh_base(
+        self,
+        *,
+        dedicated: bool = False,
+        tty: bool = False,
+    ) -> list[str]:
+        options = self._ssh_options()
+        if dedicated:
+            options.extend(
+                [
+                    "-o",
+                    "ControlMaster=no",
+                    "-o",
+                    "ControlPath=none",
+                ]
+            )
+        return [
+            "ssh",
+            *options,
+            *(["-tt"] if tty else []),
+            self.node.ssh_host,
+        ]
+
+    def open_shell(self, *, cwd: str | None = None) -> int:
+        if cwd is not None and (
+            not isinstance(cwd, str) or not cwd or "\0" in cwd
+        ):
+            raise ValueError("shell cwd must be a non-empty string")
+        if self.environment.runtime == "docker":
+            shell_setup = """
+if command -v bash >/dev/null 2>&1; then
+  shell=$(command -v bash)
+else
+  echo "msdev: bash not found; falling back to /bin/sh" >&2
+  shell=/bin/sh
+fi
+""".strip()
+        else:
+            shell_setup = 'shell="${SHELL:-/bin/sh}"'
+        if self.environment.layers:
+            shell_script = shell_setup + "\n" + """
+case "${shell##*/}" in
+  bash)
+    rc="$(mktemp "${TMPDIR:-/tmp}/msdev-shell.XXXXXX")" || exit 1
+    if [ -r "$HOME/.bashrc" ]; then
+      cat "$HOME/.bashrc" > "$rc"
+    else
+      : > "$rc"
+    fi
+    export -p >> "$rc"
+    if [ "${CONDA_PREFIX+x}" != x ]; then
+      printf '\\nunset CONDA_PREFIX CONDA_DEFAULT_ENV CONDA_PROMPT_MODIFIER CONDA_SHLVL _CE_CONDA _CE_M\\n' >> "$rc"
+    fi
+    cat >> "$rc" <<'MSDEV_SHELL_RC'
+case "$PS1" in
+  \\(*\\)\\ *) PS1="${PS1#*) }" ;;
+esac
+if [ -n "${CONDA_PROMPT_MODIFIER:-}" ]; then
+  PS1="${CONDA_PROMPT_MODIFIER}${PS1}"
+fi
+MSDEV_SHELL_RC
+    printf '\\nrm -f -- "${BASH_SOURCE[0]}"\\n' >> "$rc"
+    exec "$shell" --noprofile --rcfile "$rc" -i
+    ;;
+  *)
+    exec "$shell" -i
+    ;;
+esac
+""".strip()
+        else:
+            shell_script = shell_setup + '\nexec "$shell" -il'
+        wrapped = ["sh", "-c", shell_script]
+        if cwd is not None and self.environment.runtime == "host":
+            wrapped = [
+                "sh",
+                "-c",
+                'cd "$1" && shift && exec "$@"',
+                "msdev-shell-cwd",
+                cwd,
+                *wrapped,
+            ]
+        environments = [
+            parse_environment_spec(value)
+            for value in self.environment.layers
+        ]
+        for kind, value in reversed(environments):
+            if kind == "uv":
+                wrapped = ["uv", "run", "--project", value, "--", *wrapped]
+            elif kind == "venv":
+                activate = f"{value.rstrip('/')}/bin/activate"
+                wrapped = [
+                    "sh",
+                    "-c",
+                    f'. {shlex.quote(activate)} && exec "$@"',
+                    "msdev-shell-venv",
+                    *wrapped,
+                ]
+            elif kind == "conda":
+                wrapped = wrap_conda_command(wrapped, value)
+        if self.environment.runtime == "host" and self.node.variables:
+            wrapped = ["env", *self.node.variables, *wrapped]
+        if self.environment.runtime == "docker":
+            remote = ["docker", "exec", "-it"]
+            if cwd is not None:
+                remote.extend(["--workdir", cwd])
+            assert self.environment.container is not None
+            remote.extend([self.environment.container, *wrapped])
+        else:
+            remote = wrapped
+        completed = subprocess.run(
+            [*self._ssh_base(tty=True), shlex.join(remote)],
+            check=False,
+        )
+        return int(completed.returncode)
+
+    def connection_active(self) -> bool:
+        completed = subprocess.run(
+            [
+                "ssh",
+                "-o",
+                f"ControlPath={self.control_path}",
+                "-O",
+                "check",
+                self.node.ssh_host,
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=10,
+        )
+        return completed.returncode == 0
+
+    def connect(self) -> dict[str, Any]:
+        """Open a persistent SSH master, prompting on the caller's TTY."""
+        if self.connection_active():
+            return {
+                "connected": True,
+                "already_connected": True,
+                "persist": self.control_persist,
+            }
+        try:
+            completed = subprocess.run(
+                [
+                    "ssh",
+                    "-MNf",
+                    *self._ssh_options(),
+                    self.node.ssh_host,
+                ],
+                check=False,
+                timeout=120,
+            )
+        except subprocess.TimeoutExpired as exc:
+            raise RpcError(
+                f"SSH connection to node {self.node.name!r} timed out"
+            ) from exc
+        if completed.returncode != 0:
+            raise RpcError(
+                f"SSH authentication or connection failed for node "
+                f"{self.node.name!r} ({self.node.ssh_host})"
+            )
+        if not self.connection_active():
+            raise RpcError(
+                f"SSH master for node {self.node.name!r} did not become active"
+            )
+        return {
+            "connected": True,
+            "already_connected": False,
+            "persist": self.control_persist,
+        }
+
+    def disconnect(self) -> dict[str, Any]:
+        if not self.connection_active():
+            return {"connected": False, "already_disconnected": True}
+        completed = subprocess.run(
+            [
+                "ssh",
+                "-o",
+                f"ControlPath={self.control_path}",
+                "-O",
+                "exit",
+                self.node.ssh_host,
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=10,
+        )
+        if completed.returncode != 0 and self.connection_active():
+            raise RpcError(
+                f"failed to close SSH master for node {self.node.name!r}: "
+                f"{completed.stderr.strip()}"
+            )
+        return {"connected": False, "already_disconnected": False}
+
+    def validate(self) -> dict[str, str]:
+        completed = subprocess.run(
+            ["ssh", "-G", self.node.ssh_host],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if completed.returncode != 0:
+            raise RpcError(completed.stderr.strip() or f"invalid SSH host: {self.node.ssh_host}")
+        values: dict[str, str] = {}
+        for line in completed.stdout.splitlines():
+            key, _, value = line.partition(" ")
+            if key in {"hostname", "user", "port", "proxyjump"}:
+                values[key] = value
+        return values
+
+    def call(self, method: str, params: dict[str, Any] | None = None) -> Any:
+        try:
+            return self._call_once(method, params)
+        except DaemonUnavailableError as exc:
+            if not self.auto_bootstrap:
+                raise
+            print(
+                f"[msdev] node {self.node.name!r} is reachable but msdevd "
+                "is unavailable; bootstrapping user daemon...",
+                file=sys.stderr,
+            )
+            try:
+                bootstrap_node(self.node)
+            except Exception as bootstrap_exc:
+                raise RpcError(
+                    f"automatic bootstrap failed for node {self.node.name!r}: "
+                    f"{bootstrap_exc}. Retry with `msdev node bootstrap "
+                    f"{self.node.name}` for detailed diagnostics."
+                ) from bootstrap_exc
+            print(
+                f"[msdev] msdevd is ready on node {self.node.name!r}; "
+                "retrying request.",
+                file=sys.stderr,
+            )
+            return self._call_once(method, params)
+
+    def stream_exec(
+        self,
+        params: dict[str, Any],
+        on_output: Callable[[str, bytes], None],
+    ) -> dict[str, Any]:
+        try:
+            return self._stream_exec_once(params, on_output)
+        except DaemonUnavailableError as exc:
+            if not self.auto_bootstrap:
+                raise
+            print(
+                f"[msdev] node {self.node.name!r} is reachable but msdevd "
+                "is unavailable; bootstrapping user daemon...",
+                file=sys.stderr,
+            )
+            try:
+                bootstrap_node(self.node)
+            except Exception as bootstrap_exc:
+                raise RpcError(
+                    f"automatic bootstrap failed for node "
+                    f"{self.node.name!r}: {bootstrap_exc}. Retry with "
+                    f"`msdev node bootstrap {self.node.name}` for "
+                    "detailed diagnostics."
+                ) from bootstrap_exc
+            return self._stream_exec_once(params, on_output)
+
+    def _stream_exec_once(
+        self,
+        params: dict[str, Any],
+        on_output: Callable[[str, bytes], None],
+    ) -> dict[str, Any]:
+        self.connect()
+        request_params = dict(params)
+        if self.environment.runtime == "docker":
+            request_params["_runtime"] = {
+                "type": "docker",
+                "container": self.environment.container,
+            }
+        if self.environment.layers:
+            request_params["_environments"] = list(self.environment.layers)
+        if self.environment.runtime == "host" and self.node.variables:
+            request_params["_node_env"] = self._node_environment()
+        request_id, payload = _request(
+            "exec.stream",
+            request_params,
+            cancel_on_disconnect=True,
+        )
+        result: list[dict[str, Any]] = []
+
+        def on_line(raw: bytes) -> None:
+            decoded = _decode_stream_event(request_id, raw, on_output)
+            if decoded is not None:
+                result.append(decoded)
+
+        remote = f"{self.node.remote_bin} rpc"
+        try:
+            completed = _run_streaming_process(
+                [*self._ssh_base(dedicated=True), remote],
+                input_data=payload,
+                timeout=_rpc_timeout_seconds(
+                    "exec.stream",
+                    request_params,
+                    SSH_RPC_TIMEOUT_SECONDS,
+                ),
+                stdout_limit=MAX_RPC_RESPONSE_BYTES,
+                stderr_limit=MAX_RPC_ERROR_BYTES,
+                on_line=on_line,
+            )
+        except subprocess.TimeoutExpired as exc:
+            raise RpcError(
+                f"SSH RPC to environment {self.environment.name!r} timed out"
+            ) from exc
+        except _BoundedOutputError as exc:
+            output_name = (
+                "response" if exc.stream_name == "stdout" else "error output"
+            )
+            raise RpcError(
+                f"SSH RPC {output_name} exceeded the configured byte limit"
+            ) from exc
+        if completed.returncode != 0:
+            detail = _bounded_utf8(
+                completed.stderr.decode("utf-8", errors="replace"),
+                MAX_RPC_ERROR_BYTES,
+            )
+            raise RpcError(
+                f"SSH RPC to environment {self.environment.name!r} failed: {detail}"
+            )
+        if len(result) != 1:
+            raise RpcError("streaming RPC ended before the result event")
+        return result[0]
+
+    def _call_once(self, method: str, params: dict[str, Any] | None = None) -> Any:
+        self.connect()
+        request_params = dict(params or {})
+        if self.environment.runtime == "docker":
+            request_params["_runtime"] = {
+                "type": "docker",
+                "container": self.environment.container,
+            }
+        if self.environment.layers:
+            request_params["_environments"] = list(self.environment.layers)
+        if self.environment.runtime == "host" and self.node.variables:
+            request_params["_node_env"] = self._node_environment()
+        request_id, payload = _request(
+            method,
+            request_params,
+            cancel_on_disconnect=True,
+        )
+        remote = f"{self.node.remote_bin} rpc"
+        try:
+            completed = _run_bounded_process(
+                [
+                    *self._ssh_base(
+                        dedicated=method
+                        in {
+                            "exec",
+                            "workspace.git_status",
+                            "workspace.git_diff",
+                        }
+                    ),
+                    remote,
+                ],
+                input_data=payload,
+                timeout=_rpc_timeout_seconds(
+                    method,
+                    request_params,
+                    SSH_RPC_TIMEOUT_SECONDS,
+                ),
+                stdout_limit=MAX_RPC_RESPONSE_BYTES,
+                stderr_limit=MAX_RPC_ERROR_BYTES,
+                keep_stdin_open=True,
+            )
+        except subprocess.TimeoutExpired as exc:
+            raise RpcError(
+                f"SSH RPC to environment {self.environment.name!r} timed out"
+            ) from exc
+        except _BoundedOutputError as exc:
+            output_name = (
+                "response" if exc.stream_name == "stdout" else "error output"
+            )
+            raise RpcError(
+                f"SSH RPC {output_name} from environment {self.environment.name!r} "
+                "exceeded the configured byte limit; request a smaller result"
+            ) from exc
+        if completed.returncode != 0 and not completed.stdout:
+            stderr = completed.stderr.decode("utf-8", errors="replace").strip()
+            if completed.returncode == 127 or "not found" in stderr.lower():
+                raise DaemonUnavailableError(
+                    f"remote msdevd command is unavailable on node "
+                    f"{self.node.name!r}; run `msdev node bootstrap {self.node.name}`"
+                )
+            raise RpcError(
+                stderr
+                or f"SSH RPC to environment {self.environment.name!r} failed with "
+                f"exit code {completed.returncode}"
+            )
+        return _decode_response(request_id, completed.stdout)
+
+
+SERVICE_UNIT = """[Unit]
+Description=msdev user daemon
+After=default.target
+
+[Service]
+Type=simple
+ExecStart=%h/.local/bin/msdevd serve --socket %h/.local/share/msdevd/msdevd.sock
+Restart=on-failure
+RestartSec=2
+
+[Install]
+WantedBy=default.target
+"""
+
+
+def _package_source(package_path: Path | None = None) -> Path:
+    """Resolve an msdev package from a source checkout or user installation."""
+    if package_path is None:
+        return Path(__file__).resolve().parents[1]
+    candidate = package_path.expanduser().resolve(strict=True)
+    for source in (candidate / "src" / "msdev", candidate / "msdev", candidate):
+        if (source / "__init__.py").is_file() and (source / "daemon.py").is_file():
+            return source
+    raise ValueError(f"msdev source package not found under: {candidate}")
+
+
+def _bootstrap_archive_filter(info: tarfile.TarInfo) -> tarfile.TarInfo | None:
+    parts = info.name.split("/")
+    if (
+        any(part in {"__pycache__", ".pytest_cache"} for part in parts)
+        or any(part.endswith(".egg-info") for part in parts)
+        or info.name.endswith(".pyc")
+    ):
+        return None
+    return info
+
+
+def _cleanup_remote_archive(
+    transport: SshRpcTransport,
+    remote_archive: str,
+    package_token: str,
+) -> None:
+    """Best-effort cleanup through the already-established SSH master."""
+    command = (
+        f"rm -f -- {shlex.quote(remote_archive)}; "
+        'rm -rf -- "$HOME/.local/share/msdevd/'
+        f'package.tmp.{package_token}"'
+    )
+    try:
+        subprocess.run(
+            [
+                *transport._ssh_base(),
+                command,
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=REMOTE_CLEANUP_TIMEOUT_SECONDS,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        pass
+
+
+def bootstrap_node(node: Node, package_path: Path | None = None) -> dict[str, Any]:
+    """Deploy the pure-Python package remotely and start a user daemon."""
+    package_source = _package_source(package_path)
+
+    transport = SshRpcTransport(node, auto_bootstrap=False)
+    ssh_config = transport.validate()
+    transport.connect()
+    with tempfile.TemporaryDirectory(prefix="msdev-bootstrap-") as temp_dir:
+        archive = Path(temp_dir) / "msdev-source.tar.gz"
+        with tarfile.open(archive, "w:gz") as bundle:
+            bundle.add(
+                package_source,
+                arcname="msdev",
+                filter=_bootstrap_archive_filter,
+            )
+        package_token = uuid.uuid4().hex
+        remote_archive = f"/tmp/msdev-source-{package_token}.tar.gz"
+        try:
+            copied = subprocess.run(
+                [
+                    "scp",
+                    "-o",
+                    "ControlMaster=auto",
+                    "-o",
+                    f"ControlPersist={transport.control_persist}",
+                    "-o",
+                    f"ControlPath={transport.control_path}",
+                    str(archive),
+                    f"{node.ssh_host}:{remote_archive}",
+                ],
+                capture_output=True,
+                text=True,
+                check=False,
+                timeout=SCP_BOOTSTRAP_TIMEOUT_SECONDS,
+            )
+        except subprocess.TimeoutExpired as exc:
+            _cleanup_remote_archive(
+                transport,
+                remote_archive,
+                package_token,
+            )
+            raise RpcError(
+                f"SCP upload to node {node.name!r} timed out after "
+                f"{SCP_BOOTSTRAP_TIMEOUT_SECONDS} seconds; check node "
+                f"connectivity and retry `msdev node bootstrap {node.name}`"
+            ) from exc
+        if copied.returncode != 0:
+            _cleanup_remote_archive(
+                transport,
+                remote_archive,
+                package_token,
+            )
+            raise RuntimeError(copied.stderr.strip() or "failed to upload msdev wheel")
+
+        unit_b64 = base64.b64encode(SERVICE_UNIT.encode("utf-8")).decode("ascii")
+        wrapper = """#!/usr/bin/env bash
+set -e
+export PYTHONPATH="${HOME}/.local/share/msdevd/package${PYTHONPATH:+:${PYTHONPATH}}"
+exec python3 -m msdev.daemon "$@"
+"""
+        wrapper_b64 = base64.b64encode(wrapper.encode("utf-8")).decode("ascii")
+        script = f"""
+set -eu
+bootstrap_dir="$HOME/.local/share/msdevd"
+package_root="$bootstrap_dir/package"
+package_tmp="$bootstrap_dir/package.tmp.{package_token}"
+lock_path="$bootstrap_dir/bootstrap.lock"
+cleanup_bootstrap() {{
+  rm -f -- {shlex.quote(remote_archive)}
+  rm -rf -- "$package_tmp"
+}}
+trap cleanup_bootstrap EXIT
+trap 'exit 1' HUP INT TERM
+mkdir -p "$bootstrap_dir"
+exec 9>"$lock_path"
+if ! flock -w 30 9; then
+  echo "timed out waiting for msdev bootstrap lock" >&2
+  exit 75
+fi
+rm -rf "$package_tmp"
+mkdir -p "$package_tmp" "$HOME/.config/systemd/user" "$HOME/.local/bin"
+tar -xzf {shlex.quote(remote_archive)} -C "$package_tmp"
+rm -f {shlex.quote(remote_archive)}
+rm -rf "$package_root.old"
+if [ -d "$package_root" ]; then mv "$package_root" "$package_root.old"; fi
+mv "$package_tmp" "$package_root"
+printf %s {shlex.quote(wrapper_b64)} | base64 -d > "$HOME/.local/bin/msdevd"
+chmod 700 "$HOME/.local/bin/msdevd"
+printf %s {shlex.quote(unit_b64)} | base64 -d > "$HOME/.config/systemd/user/msdevd.service"
+# Stop stale user daemons from older releases that selected a session-specific
+# socket through XDG_RUNTIME_DIR or TMPDIR.
+python3 - <<'PY'
+import os
+import signal
+import time
+from pathlib import Path
+
+current = os.getpid()
+stale_pids = []
+for entry in Path("/proc").iterdir():
+    if not entry.name.isdigit() or int(entry.name) == current:
+        continue
+    try:
+        if entry.stat().st_uid != os.getuid():
+            continue
+        command = [part for part in (entry / "cmdline").read_bytes().split(b"\\0") if part]
+    except (FileNotFoundError, PermissionError, ProcessLookupError):
+        continue
+    try:
+        module_index = command.index(b"-m")
+    except ValueError:
+        continue
+    if command[module_index + 1:module_index + 3] == [b"msdev.daemon", b"serve"]:
+        try:
+            os.kill(int(entry.name), signal.SIGTERM)
+            stale_pids.append(int(entry.name))
+        except (ProcessLookupError, PermissionError):
+            pass
+for _ in range(20):
+    if not any(Path(f"/proc/{{pid}}").exists() for pid in stale_pids):
+        break
+    time.sleep(0.1)
+PY
+rm -f "$HOME/.local/share/msdevd/msdevd.sock"
+flock -u 9
+exec 9>&-
+if command -v systemctl >/dev/null 2>&1 && systemctl --user daemon-reload >/dev/null 2>&1; then
+  systemctl --user enable msdevd.service >/dev/null
+  systemctl --user restart msdevd.service
+else
+  socket_path="$HOME/.local/share/msdevd/msdevd.sock"
+  if [ ! -S "$socket_path" ]; then
+    nohup "$HOME/.local/bin/msdevd" serve --socket "$socket_path" \
+      > "$HOME/.local/share/msdevd/daemon.log" 2>&1 < /dev/null &
+  fi
+fi
+"""
+        try:
+            try:
+                installed = subprocess.run(
+                    [*transport._ssh_base(), f"bash -lc {shlex.quote(script)}"],
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                    timeout=180,
+                )
+            except subprocess.TimeoutExpired as exc:
+                raise RpcError(
+                    f"remote msdev installation on node {node.name!r} "
+                    "timed out after 180 seconds; check node capacity and "
+                    f"retry `msdev node bootstrap {node.name}`"
+                ) from exc
+        finally:
+            _cleanup_remote_archive(
+                transport,
+                remote_archive,
+                package_token,
+            )
+        if installed.returncode != 0:
+            raise RuntimeError(installed.stderr.strip() or installed.stdout.strip())
+
+    last_error: Exception | None = None
+    ping = None
+    for _ in range(50):
+        try:
+            ping = transport.call("ping")
+            break
+        except DaemonUnavailableError as exc:
+            last_error = exc
+            time.sleep(0.1)
+    if ping is None:
+        raise RpcError(
+            f"msdevd did not become ready on node {node.name!r}: {last_error}"
+        )
+    return {"ssh": ssh_config, "daemon": ping}