offbox 0.1.0__py3-none-any.whl

offbox/host/create.py

@@ -0,0 +1,198 @@
+import logging
+import shutil
+from collections.abc import Callable
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pydantic import field_validator
+
+from offbox import console
+from offbox.config import HostPreset, Model, UserConfig, load_config
+from offbox.drivers.base import HostVariables
+from offbox.drivers.tofu import TofuDriver
+from offbox.errors import CommandFailed, OffboxError, PreconditionError, ProviderInterrupted
+from offbox.host import host_operation, public_key
+from offbox.host.cloudinit import generate_host_key, render_cloud_init
+from offbox.host.ready import Clock, SystemClock, tcp_probe, wait_ready
+from offbox.paths import Paths
+from offbox.providers import copy_provider, credentials_env, discover_provider, validate_name
+from offbox.runner import Runner
+from offbox.ssh import close_master, default_ssh_key, ensure_include, host_key_path, known_host, ssh_argv, sync_config, write_private
+from offbox.state import HostState, StateStore
+from offbox.timings import stage
+from offbox.tools import ToolDiscovery
+
+logger = logging.getLogger(__name__)
+_NAME_TAKEN = ("uniqueness_error", "alreadyExists", "already exists")
+
+
+def _name_taken(exc: CommandFailed) -> bool:
+    return any(marker in exc.stderr_tail for marker in _NAME_TAKEN)
+
+
+class CreateOptions(Model):
+    name: str = "main"
+    provider: str | None = None
+    size: str | None = None
+    region: str | None = None
+    ssh_key: Path | None = None
+    auto: bool = False
+    idle_seconds: int | None = None
+
+    @field_validator("name")
+    @classmethod
+    def check_name(cls, value: str) -> str:
+        return validate_name(value)
+
+
+def _remove_unrecorded(directory: Path) -> None:
+    try:
+        shutil.rmtree(directory)
+    except OSError:
+        logger.exception("Cannot remove unrecorded host directory %s", directory)
+        console.warn(f"Remove {directory} before retrying host creation.")
+
+
+def create_host_locked(
+    paths: Paths,
+    runner: Runner,
+    options: CreateOptions,
+    *,
+    executable: Path | None = None,
+    clock: Clock | None = None,
+    probe: Callable[[str, int, float], bool] = tcp_probe,
+    ssh_config: Path | None = None,
+) -> HostState:
+    store = StateStore(paths.state)
+    if options.name in store.read().hosts:
+        raise PreconditionError(f"Host {options.name} already exists", f"Run offbox host destroy {options.name} or choose another name.")
+    timer = clock or SystemClock()
+    started = timer.now()
+    user = load_config(paths.user_config, UserConfig)
+    preset = user.hosts.get(options.name, HostPreset())
+    provider = discover_provider(options.provider or preset.provider, paths)
+    if provider.manifest.driver != "tofu":
+        raise PreconditionError("This provider registers existing hosts", "Run offbox host init USER@ADDRESS.")
+    credentials = credentials_env(provider.manifest, paths)
+    key = (options.ssh_key or preset.ssh_key or default_ssh_key()).expanduser().resolve()
+    authorized_key = public_key(runner, key)
+    binary = executable or ToolDiscovery(runner, paths.cache).check("tofu").path
+    defaults = provider.manifest.defaults
+    size = options.size or preset.size or defaults.size
+    region = options.region or preset.region or defaults.region
+    image = preset.image or defaults.image
+    if not size or not region or not image:
+        raise PreconditionError("Provider size, region and image are required", "Set provider defaults or a matching host preset.")
+    directory = paths.state / "hosts" / options.name
+    workspace = directory / "tofu"
+    created = recorded = False
+    try:
+        with store.transaction() as state:
+            if options.name in state.hosts:
+                raise PreconditionError(
+                    f"Host {options.name} already exists", f"Run offbox host destroy {options.name} or choose another name."
+                )
+            if directory.exists():
+                raise PreconditionError(
+                    f"Host directory already exists: {directory}",
+                    "Inspect the existing directory before removing it or choose another host name.",
+                )
+            directory.mkdir(parents=True, mode=0o700)
+            created = True
+            digest = copy_provider(provider, workspace)
+            host = HostState(
+                name=options.name,
+                provider=provider.manifest.name,
+                address="",
+                ssh_user="offbox",
+                created_at=datetime.now(UTC),
+                provider_dir=workspace,
+                provider_files_hash=digest,
+                known_hosts_path=directory / "known_hosts",
+                ssh_key=key,
+                phase="creating",
+                auto=options.auto,
+                idle_seconds=options.idle_seconds,
+            )
+            state.hosts[host.name] = host
+        recorded = True
+    finally:
+        if created and not recorded:
+            _remove_unrecorded(directory)
+    cleanup = f"Run offbox host destroy {host.name} to clean up; its provider workspace was retained."
+    try:
+        private, public = generate_host_key(runner, host_key_path(host))
+        write_private(host.known_hosts_path, known_host(host.name, public))
+        variables = HostVariables(
+            name=f"offbox-{host.name}",
+            size=size,
+            region=region,
+            image=image,
+            ssh_public_key=authorized_key,
+            user_data=render_cloud_init(authorized_key, private, public),
+            labels={"offbox": "1", "offbox-host": host.name},
+        )
+        try:
+            output = TofuDriver(runner, workspace, paths.cache, credentials, binary).create(variables)
+        except CommandFailed as exc:
+            if _name_taken(exc):
+                raise PreconditionError(
+                    f"Cloud resources named offbox-{host.name} already exist in the provider project",
+                    "Delete them in the provider console or choose another host name, "
+                    f"then run offbox host destroy {host.name} to clear the local record.",
+                ) from exc
+            raise
+        host.address = output.ip
+        host.ssh_user = output.ssh_user
+        host.phase = "initializing"
+        with store.transaction() as state:
+            state.hosts[host.name] = host
+            sync_config(paths, state)
+            ensure_include(paths, ssh_config)
+        wait_ready(host.name, host.address, host.ssh_port, runner, paths.log_file(), clock=timer, probe=probe)
+        # Readiness polling can authenticate before bootstrap adds the Docker group.
+        # A multiplexed connection retains that login's groups until it is replaced.
+        with stage("SSH refresh"):
+            close_master(runner, host.name)
+            runner.run(ssh_argv(host.name, ["docker", "info", "--format", "{{.ServerVersion}}"]), timeout=10)
+        host.phase = "ready"
+        with store.transaction() as state:
+            state.hosts[host.name] = host
+            if state.default_host is None and not host.auto:
+                state.default_host = host.name
+    except ProviderInterrupted as exc:
+        if exc.killed:
+            host.create_interrupted = True
+            with store.transaction() as state:
+                if state.hosts.get(host.name) is not None:
+                    state.hosts[host.name] = host
+        fix = (
+            f"Run offbox host destroy {host.name}, then check the provider console for resources labelled "
+            f"offbox-host={host.name} and delete any that remain."
+            if exc.killed
+            else cleanup
+        )
+        raise OffboxError(f"Creating host {host.name} failed: {exc.message}", fix, exc.exit_code) from exc
+    except OffboxError as exc:
+        raise OffboxError(f"Creating host {host.name} failed: {exc.message}", cleanup, exc.exit_code) from exc
+    except KeyboardInterrupt as exc:
+        raise OffboxError(f"Creating host {host.name} was interrupted", cleanup, 130) from exc
+    finally:
+        host_key_path(host).unlink(missing_ok=True)
+        host_key_path(host).with_name(host_key_path(host).name + ".pub").unlink(missing_ok=True)
+    console.success(f"Host {host.name}: {host.address} ({timer.now() - started:.1f}s)")
+    return host
+
+
+def create_host(
+    paths: Paths,
+    runner: Runner,
+    options: CreateOptions,
+    *,
+    executable: Path | None = None,
+    clock: Clock | None = None,
+    probe: Callable[[str, int, float], bool] = tcp_probe,
+    ssh_config: Path | None = None,
+) -> HostState:
+    with host_operation(paths, options.name):
+        return create_host_locked(paths, runner, options, executable=executable, clock=clock, probe=probe, ssh_config=ssh_config)

offbox/host/destroy.py

@@ -0,0 +1,151 @@
+import logging
+import re
+import shutil
+from collections.abc import Callable
+from functools import partial
+from pathlib import Path
+
+from offbox import console
+from offbox.drivers.none import NoneDriver
+from offbox.drivers.tofu import TofuDriver
+from offbox.errors import OffboxError, PreconditionError, ProviderInterrupted, UsageError
+from offbox.host import host_operation, select_host
+from offbox.host.activity import forget as forget_activity
+from offbox.host.usage import describe, exclusive_usage, holders
+from offbox.host.warnings import warn_interrupted
+from offbox.naming import Worktree
+from offbox.paths import Paths
+from offbox.providers import credentials_env, load_provider, validate_name
+from offbox.runner import Runner
+from offbox.ssh import close_master, sync_config
+from offbox.state import EnvState, HostState, StateStore
+from offbox.sync import SyncManager
+from offbox.tools import ToolDiscovery
+
+logger = logging.getLogger(__name__)
+
+
+def _best_effort(action: Callable[[], None], warning: str) -> None:
+    try:
+        action()
+    except OffboxError as exc:
+        logger.warning("%s", warning, exc_info=True)
+        console.warn(f"{warning} ({exc.message})")
+
+
+def _release_local(runner: Runner, cache: Path, host: HostState, envs: dict[str, EnvState], *, strict: bool) -> None:
+    sync = SyncManager(runner, cache=cache)
+    for env in envs.values():
+        work = Worktree(root=env.worktree_path, name=env.name)
+        if strict:
+            sync.terminate(work, host)
+        else:
+            _best_effort(
+                partial(sync.terminate, work, host),
+                f"Cannot stop source sync for {env.name}; afterwards run mutagen sync list and terminate offbox-{env.name} if it remains",
+            )
+    if not host.address:
+        return
+    if strict:
+        close_master(runner, host.name)
+    else:
+        _best_effort(
+            partial(close_master, runner, host.name),
+            f"Cannot close the SSH master for offbox-{host.name}; stop that local ssh process manually",
+        )
+
+
+def _destroy_cloud(runner: Runner, host: HostState, workspace: Path, cache: Path, credentials: dict[str, str], binary: Path) -> None:
+    try:
+        TofuDriver(runner, workspace, cache, credentials, binary).destroy()
+    except ProviderInterrupted as exc:
+        fix = f"Run offbox host destroy {host.name} again."
+        if exc.killed:
+            fix = (
+                f"Run offbox host destroy {host.name} again, then check the provider console for resources labelled "
+                f"offbox-host={host.name} and delete any that remain."
+            )
+        raise ProviderInterrupted(f"Destroying host {host.name} was interrupted", fix, killed=exc.killed) from exc
+
+
+def destroy_host_locked(
+    paths: Paths, runner: Runner, name: str | None = None, *, executable: Path | None = None, forget: bool = False
+) -> None:
+    host = select_host(paths, name)
+    validate_name(host.name)
+    directory = paths.state / "hosts" / host.name
+    expected = directory / ("provider" if host.provider == "ssh" else "tofu")
+    if host.provider_dir is None or host.provider_dir.resolve() != expected.resolve() or directory.is_symlink() or expected.is_symlink():
+        raise PreconditionError(
+            "Host provider workspace is missing or unsafe", "Restore the host's saved provider workspace before destroying it."
+        )
+    provider = load_provider(host.provider_dir)
+    existing = provider.manifest.driver == "none"
+    if forget and not existing:
+        raise UsageError(
+            f"Host {host.name} is a cloud host; --forget would leave it running and billing",
+            f"Run offbox host destroy {host.name} --yes to delete it.",
+        )
+    credentials = credentials_env(provider.manifest, paths)
+    binary = None if existing else executable or ToolDiscovery(runner, paths.cache).check("tofu").path
+    store = StateStore(paths.state)
+    snapshot = store.read()
+    if snapshot.hosts.get(host.name) != host:
+        raise PreconditionError("Host changed while preparing destruction", "Run host destroy again after other operations finish.")
+    envs = {name: env for name, env in snapshot.envs.items() if env.host == host.name}
+    for env in envs.values():
+        if re.fullmatch(r"[a-z0-9][a-z0-9-]{0,36}", env.name) is None:
+            raise PreconditionError("Invalid stored environment name", "Repair the environment state before destroying its host.")
+    _release_local(runner, paths.cache, host, envs, strict=existing and not forget)
+    if existing and not forget:
+        NoneDriver(runner, host).destroy()
+        _best_effort(
+            partial(close_master, runner, host.name),
+            f"Cannot close the SSH master for offbox-{host.name}; stop that local ssh process manually",
+        )
+    elif not existing:
+        if binary is None:
+            raise PreconditionError("Provider tool is unavailable", "Install OpenTofu or Terraform and retry.")
+        _destroy_cloud(runner, host, host.provider_dir, paths.cache, credentials, binary)
+    with store.transaction() as state:
+        if state.hosts.get(host.name) != host or {name: env for name, env in state.envs.items() if env.host == host.name} != envs:
+            logger.warning("host %s changed during destruction", host.name)
+            console.warn(
+                f"Host {host.name} changed while it was being destroyed; removing its record and environments anyway. "
+                "Review offbox list and mutagen sync list for environments added meanwhile."
+            )
+        state.hosts.pop(host.name, None)
+        state.envs = {name: env for name, env in state.envs.items() if env.host != host.name}
+        if state.default_host == host.name:
+            state.default_host = next((name for name, item in state.hosts.items() if item.phase == "ready" and not item.auto), None)
+        sync_config(paths, state)
+    for env_name in envs:
+        forget_activity(paths, env_name)
+    if host.create_interrupted:
+        warn_interrupted(paths, host.name)
+    try:
+        shutil.rmtree(directory)
+    except OSError as exc:
+        raise PreconditionError(
+            f"Host destroyed, but cannot remove {directory}", "Remove the obsolete local host directory manually."
+        ) from exc
+    if forget:
+        console.warn(
+            f"Forgot host {host.name} without contacting {host.address or 'the server'}. Still on the server: "
+            "offbox-labelled Docker containers and volumes, the offbox Docker network, /srv/offbox, /var/lib/offbox, "
+            "the offbox user, /home/offbox/.ssh/authorized_keys and its passwordless sudo rule /etc/sudoers.d/offbox."
+        )
+        console.success(f"Forgot host {host.name}")
+        return
+    console.success(f"Destroyed host {host.name}")
+
+
+def destroy_host(paths: Paths, runner: Runner, name: str | None = None, *, executable: Path | None = None, forget: bool = False) -> None:
+    host = select_host(paths, name)
+    with exclusive_usage(paths, host.name) as free:
+        if not free:
+            raise PreconditionError(
+                f"Host {host.name} is in use by: {describe(holders(paths, host.name))}", "Wait for those commands to finish, then retry."
+            )
+        with host_operation(paths, host.name):
+            destroy_host_locked(paths, runner, host.name, executable=executable, forget=forget)

offbox/host/init.py

@@ -0,0 +1,196 @@
+import base64
+import binascii
+import hashlib
+import re
+import shlex
+from collections.abc import Callable
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pydantic import Field, field_validator
+
+from offbox import DEFAULT_IMAGE, __version__, console
+from offbox.config import Model
+from offbox.errors import OffboxError, PreconditionError, UsageError
+from offbox.host import host_operation, public_key
+from offbox.host.cloudinit import bootstrap_script
+from offbox.host.ready import Clock, tcp_probe, wait_ready
+from offbox.paths import Paths
+from offbox.providers import copy_provider, discover_provider, validate_name
+from offbox.runner import Runner
+from offbox.ssh import default_ssh_key, ensure_include, known_host, quote_path, sync_config, validate_address, validate_user, write_private
+from offbox.state import HostState, StateStore
+
+
+class InitOptions(Model):
+    target: str
+    name: str = "main"
+    port: int = Field(default=22, ge=1, le=65535)
+    ssh_key: Path | None = None
+    accept_host_key: bool = False
+    host_key_fingerprint: str | None = None
+
+    @field_validator("name")
+    @classmethod
+    def check_name(cls, value: str) -> str:
+        return validate_name(value)
+
+    @field_validator("host_key_fingerprint")
+    @classmethod
+    def check_fingerprint(cls, value: str | None) -> str | None:
+        if value is not None and re.fullmatch(r"SHA256:[A-Za-z0-9+/]{43}", value) is None:
+            raise UsageError("Invalid host key fingerprint", "Pass the SHA256:... value printed by ssh-keygen -lf on the server.")
+        return value
+
+
+def _init_host(
+    paths: Paths,
+    runner: Runner,
+    options: InitOptions,
+    *,
+    confirm: Callable[[str], bool] | None = None,
+    ssh_config: Path | None = None,
+    clock: Clock | None = None,
+    probe: Callable[[str, int, float], bool] = tcp_probe,
+) -> HostState:
+    if not options.accept_host_key and options.host_key_fingerprint is None and confirm is None:
+        raise PreconditionError(
+            "Host key verification needs a TTY, --host-key-fingerprint or --accept-host-key",
+            "Pass --host-key-fingerprint SHA256:... from ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub on the server.",
+        )
+    if options.target.count("@") != 1:
+        raise PreconditionError("Expected USER@ADDRESS", "Run offbox host init USER@ADDRESS.")
+    owner, address = options.target.split("@", 1)
+    validate_user(owner)
+    address = validate_address(address.strip("[]"))
+    key = (options.ssh_key or default_ssh_key()).expanduser().resolve()
+    scan = runner.run(["ssh-keyscan", "-T", "5", "-p", str(options.port), "-t", "ed25519", address], timeout=10)
+    keys = {
+        " ".join(line.split()[1:3])
+        for line in scan.stdout.splitlines()
+        if line.strip() and not line.startswith("#") and len(line.split()) >= 3
+    }
+    if len(keys) != 1:
+        raise PreconditionError("Cannot determine a unique ed25519 host key", "Check the SSH address and its advertised host keys.")
+    scanned_key = keys.pop()
+    pinned = known_host(options.name, scanned_key)
+    try:
+        digest = base64.b64encode(hashlib.sha256(base64.b64decode(scanned_key.split()[1], validate=True)).digest()).decode().rstrip("=")
+    except (ValueError, binascii.Error) as exc:
+        raise PreconditionError("Invalid SSH host key", "Check the server's ed25519 key.") from exc
+    fingerprint = f"SHA256:{digest}"
+    console.info(f"Host {address} fingerprint {fingerprint}")
+    if options.host_key_fingerprint is not None:
+        if options.host_key_fingerprint != fingerprint:
+            raise PreconditionError(
+                f"Host key fingerprint mismatch for {address}: expected {options.host_key_fingerprint}, got {fingerprint}",
+                "Verify the server identity with ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub on the server before retrying.",
+            )
+    elif options.accept_host_key:
+        console.warn(f"Trusting {fingerprint} without verification; prefer --host-key-fingerprint {fingerprint}.")
+    elif confirm is None or not confirm(f"Trust {fingerprint} for {address}?"):
+        raise PreconditionError("Host key was not accepted", "Verify the fingerprint and rerun offbox host init.")
+    authorized_key = public_key(runner, key)
+    directory = paths.state / "hosts" / options.name
+    provider = discover_provider("ssh", paths)
+    if provider.manifest.driver != "none":
+        raise PreconditionError("The ssh provider must use driver none", "Correct the ssh provider manifest.")
+    store = StateStore(paths.state)
+    with store.transaction() as state:
+        previous = state.hosts.get(options.name)
+        if previous is not None:
+            if previous.provider != "ssh" or previous.address != address or previous.ssh_port != options.port or previous.phase == "ready":
+                raise PreconditionError(
+                    f"Host {options.name} already exists", f"Run offbox host destroy {options.name} or choose another name."
+                )
+            if previous.known_hosts_path.read_text(encoding="utf-8") != pinned:
+                raise PreconditionError(
+                    "The existing host key has changed", "Verify the server identity before changing its stored host key."
+                )
+            host = previous
+            host.ssh_key = key
+        else:
+            if directory.exists():
+                raise PreconditionError(
+                    f"Host directory already exists: {directory}", "Inspect the existing directory or choose another name."
+                )
+            directory.mkdir(parents=True, mode=0o700)
+            workspace = directory / "provider"
+            digest_files = copy_provider(provider, workspace)
+            host = HostState(
+                name=options.name,
+                provider="ssh",
+                address=address,
+                ssh_user="offbox",
+                ssh_port=options.port,
+                created_at=datetime.now(UTC),
+                provider_dir=workspace,
+                provider_files_hash=digest_files,
+                known_hosts_path=directory / "known_hosts",
+                ssh_key=key,
+                phase="initializing",
+            )
+        state.hosts[host.name] = host
+        write_private(host.known_hosts_path, pinned)
+        sync_config(paths, state)
+        ensure_include(paths, ssh_config)
+    command = ["sudo", "-n", "env", "OFFBOX_HARDEN=0", f"OFFBOX_VERSION={__version__}", "sh", "-s", "--", DEFAULT_IMAGE, authorized_key]
+    try:
+        runner.run(
+            [
+                "ssh",
+                "-o",
+                "BatchMode=yes",
+                "-o",
+                "StrictHostKeyChecking=yes",
+                "-o",
+                "HostKeyAlgorithms=ssh-ed25519",
+                "-o",
+                f"HostKeyAlias=offbox-{host.name}",
+                "-o",
+                f"UserKnownHostsFile={quote_path(host.known_hosts_path)}",
+                "-o",
+                "IdentitiesOnly=yes",
+                "-o",
+                "ForwardAgent=no",
+                "-o",
+                "ForwardX11=no",
+                "-i",
+                str(key),
+                "-p",
+                str(options.port),
+                f"{owner}@{address}",
+                "--",
+                shlex.join(command),
+            ],
+            input=bootstrap_script(),
+            timeout=900,
+        )
+        wait_ready(host.name, host.address, host.ssh_port, runner, paths.log_file(), clock=clock, probe=probe)
+        host.phase = "ready"
+        with store.transaction() as state:
+            state.hosts[host.name] = host
+            if state.default_host is None:
+                state.default_host = host.name
+    except OffboxError as exc:
+        raise OffboxError(
+            f"Initialising host {host.name} failed: {exc.message}",
+            "Correct the reported problem and rerun the same host init command; its registration was retained.",
+            exc.exit_code,
+        ) from exc
+    console.success(f"Host {host.name}: {host.address}")
+    return host
+
+
+def init_host(
+    paths: Paths,
+    runner: Runner,
+    options: InitOptions,
+    *,
+    confirm: Callable[[str], bool] | None = None,
+    ssh_config: Path | None = None,
+    clock: Clock | None = None,
+    probe: Callable[[str, int, float], bool] = tcp_probe,
+) -> HostState:
+    with host_operation(paths, options.name):
+        return _init_host(paths, runner, options, confirm=confirm, ssh_config=ssh_config, clock=clock, probe=probe)

offbox/host/ready.py

@@ -0,0 +1,106 @@
+import logging
+import socket
+import time
+from collections.abc import Callable
+from pathlib import Path
+from typing import NoReturn, Protocol
+
+from offbox.errors import ConnectionError_, OffboxError, ProviderError
+from offbox.runner import Runner
+from offbox.ssh import ssh_argv
+from offbox.timings import measured
+
+logger = logging.getLogger(__name__)
+
+BOOTSTRAP_FAILED = 7
+BOOTSTRAP_TAIL_LINES = 15
+READY_PROBE = (
+    "test -f /var/lib/offbox/ready && exit 0; "
+    f"test -f /var/lib/offbox/failed && {{ head -c 2000 /var/lib/offbox/failed; exit {BOOTSTRAP_FAILED}; }}; "
+    "exit 1"
+)
+
+
+class Clock(Protocol):
+    def now(self) -> float: ...
+    def sleep(self, seconds: float) -> None: ...
+
+
+class SystemClock:
+    def now(self) -> float:
+        return time.monotonic()
+
+    def sleep(self, seconds: float) -> None:
+        time.sleep(seconds)
+
+
+def tcp_probe(address: str, port: int, timeout: float) -> bool:
+    try:
+        with socket.create_connection((address, port), timeout=timeout):
+            return True
+    except OSError:
+        return False
+
+
+@measured("Host readiness")
+def wait_ready(
+    name: str,
+    address: str,
+    port: int,
+    runner: Runner,
+    log_path: Path,
+    *,
+    clock: Clock | None = None,
+    probe: Callable[[str, int, float], bool] = tcp_probe,
+    tcp_timeout: float = 300,
+    ready_timeout: float = 900,
+) -> None:
+    timer = clock or SystemClock()
+
+    def diagnostics() -> str:
+        try:
+            diagnostic = runner.run(
+                ssh_argv(name, ["sudo", "-n", "tail", "-n", "40", "/var/log/cloud-init-output.log"], options=["-o", "ConnectTimeout=3"]),
+                check=False,
+                timeout=10,
+            )
+            logger.error("host bootstrap diagnostics: %s\n%s", diagnostic.stdout, diagnostic.stderr)
+            return diagnostic.stdout
+        except OffboxError:
+            logger.exception("Cannot retrieve bootstrap diagnostics")
+            return ""
+
+    def fail() -> NoReturn:
+        diagnostics()
+        raise ConnectionError_(f"Host {name} did not become ready", f"Run offbox host ssh {name}; inspect {log_path}.")
+
+    def failed(marker: str) -> NoReturn:
+        tail = "\n".join(diagnostics().splitlines()[-BOOTSTRAP_TAIL_LINES:])
+        details = marker.strip() or "no details recorded"
+        raise ProviderError(
+            f"Host {name} bootstrap failed ({details})" + (f"\n{tail}" if tail else ""),
+            f"Run offbox host ssh {name} and inspect /var/log/cloud-init-output.log and {log_path}.",
+        )
+
+    deadline = timer.now() + tcp_timeout
+    while not probe(address, port, min(3, max(0.1, deadline - timer.now()))):
+        if timer.now() >= deadline:
+            fail()
+        timer.sleep(min(3, max(0, deadline - timer.now())))
+    deadline = timer.now() + ready_timeout
+    while timer.now() < deadline:
+        try:
+            result = runner.run(
+                ssh_argv(name, ["sh", "-c", READY_PROBE], options=["-o", "ConnectTimeout=3"]),
+                check=False,
+                timeout=min(10, deadline - timer.now()),
+            )
+        except OffboxError:
+            logger.debug("Host SSH not ready", exc_info=True)
+        else:
+            if result.returncode == 0:
+                return
+            if result.returncode == BOOTSTRAP_FAILED:
+                failed(result.stdout)
+        timer.sleep(min(3, max(0, deadline - timer.now())))
+    fail()