offbox 0.1.0__py3-none-any.whl

offbox/host/__init__.py

@@ -0,0 +1,89 @@
+import fcntl
+from collections.abc import Generator
+from contextlib import contextmanager
+from pathlib import Path
+
+from offbox.config import UserConfig, default_host, load_config
+from offbox.errors import PreconditionError
+from offbox.host.ready import Clock, SystemClock
+from offbox.locking import acquire_lock
+from offbox.paths import Paths
+from offbox.providers import validate_name
+from offbox.runner import Runner
+from offbox.state import HostState, StateStore
+
+
+def select_host(paths: Paths, name: str | None = None) -> HostState:
+    state = StateStore(paths.state).read()
+    selected = name or default_host(load_config(paths.user_config, UserConfig), state.default_host)
+    if selected is None:
+        raise PreconditionError("No default host", "Run offbox setup, or offbox host use NAME.")
+    validate_name(selected)
+    if selected not in state.hosts:
+        raise PreconditionError(f"Unknown host: {selected}", "Run offbox host list or create this host.")
+    return state.hosts[selected]
+
+
+def public_key(runner: Runner, key: Path) -> str:
+    derived = runner.run(["ssh-keygen", "-y", "-P", "", "-f", str(key)], check=False, timeout=5)
+    if derived.returncode == 0:
+        content = derived.stdout.strip()
+        sidecar = key.with_name(key.name + ".pub")
+        try:
+            if sidecar.exists() and sidecar.read_text(encoding="utf-8").split()[:2] != content.split()[:2]:
+                raise PreconditionError(
+                    "SSH public key sidecar does not match the private key", f"Regenerate {sidecar} with ssh-keygen -y -f {key}."
+                )
+        except (OSError, UnicodeError) as exc:
+            raise PreconditionError(f"Cannot read {sidecar}", "Correct the SSH public key file permissions and encoding.") from exc
+    else:
+        path = key.with_name(key.name + ".pub")
+        if not path.is_file():
+            raise PreconditionError(f"SSH key is unavailable: {key}", f"Generate the key or load it with ssh-add {key}.")
+        actual = runner.run(["ssh-keygen", "-lf", str(path)], check=False, timeout=5)
+        expected = runner.run(["ssh-keygen", "-lf", str(key)], check=False, timeout=5) if key.is_file() else actual
+        agent = runner.run(["ssh-add", "-l"], check=False, timeout=5)
+        expected_fields, actual_fields = expected.stdout.split(), actual.stdout.split()
+        if (
+            expected.returncode
+            or actual.returncode
+            or agent.returncode
+            or len(expected_fields) < 2
+            or len(actual_fields) < 2
+            or expected_fields[1] != actual_fields[1]
+            or expected_fields[1] not in agent.stdout
+        ):
+            raise PreconditionError(
+                "SSH public key does not match an available identity", f"Regenerate {path} and load the matching key with ssh-add {key}."
+            )
+        try:
+            content = path.read_text(encoding="utf-8").strip()
+        except (OSError, UnicodeError) as exc:
+            raise PreconditionError(f"Cannot read public key for {key}", "Generate its .pub file with ssh-keygen -y.") from exc
+    fields = content.split()
+    if len(content.splitlines()) != 1 or len(fields) < 2 or not fields[0].startswith(("ssh-", "ecdsa-", "sk-")):
+        raise PreconditionError("Invalid SSH public key", "Generate a valid SSH key with ssh-keygen.")
+    return content
+
+
+@contextmanager
+def host_operation(paths: Paths, name: str, *, wait_timeout: float = 0, clock: Clock | None = None) -> Generator[None]:
+    validate_name(name)
+    directory = paths.state / "host-locks"
+    try:
+        directory.mkdir(parents=True, exist_ok=True, mode=0o700)
+        handle = (directory / f"{name}.lock").open("a", encoding="utf-8")
+    except OSError as exc:
+        raise PreconditionError("Cannot open host operation lock", "Check state directory permissions.") from exc
+    with handle:
+        acquire_lock(
+            handle.fileno(),
+            timeout=wait_timeout,
+            clock=clock or SystemClock(),
+            message=f"A lifecycle operation is already running for {name}",
+            fix="Wait for that operation to finish, then retry.",
+        )
+        try:
+            yield
+        finally:
+            fcntl.flock(handle.fileno(), fcntl.LOCK_UN)

offbox/host/activity.py

@@ -0,0 +1,58 @@
+import logging
+import os
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+
+from offbox.errors import PreconditionError
+from offbox.paths import Paths
+from offbox.state import EnvState
+
+logger = logging.getLogger(__name__)
+_ENV_NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,36}")
+
+
+def activity_path(paths: Paths, env: str) -> Path:
+    if _ENV_NAME.fullmatch(env) is None:
+        raise PreconditionError("Invalid environment name", "Repair the environment state.")
+    return paths.state / "activity" / env
+
+
+def touch(paths: Paths, env: str, now: datetime) -> None:
+    """Record activity without locks; a failure is logged and never stops the command."""
+    path = activity_path(paths, env)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        path.touch(mode=0o600)
+        stamp = now.timestamp()
+        os.utime(path, (stamp, stamp))
+    except OSError:
+        logger.warning("cannot record activity for %s", env, exc_info=True)
+
+
+def last_activity(paths: Paths, env: EnvState) -> datetime:
+    try:
+        return max(env.created_at, datetime.fromtimestamp(activity_path(paths, env.name).stat().st_mtime, UTC))
+    except OSError:
+        return env.created_at
+
+
+def forget(paths: Paths, env: str) -> None:
+    try:
+        activity_path(paths, env).unlink(missing_ok=True)
+    except OSError:
+        logger.warning("cannot remove activity record for %s", env, exc_info=True)
+
+
+def worktree_gone(root: Path) -> bool:
+    """True only for a removed worktree: the path is missing while its parent exists. Unreadable or unmounted paths count as present."""
+    try:
+        root.stat()
+        return False
+    except (FileNotFoundError, NotADirectoryError):
+        try:
+            return root.parent.is_dir()
+        except OSError:
+            return False
+    except OSError:
+        return False

offbox/host/auto.py

@@ -0,0 +1,322 @@
+import logging
+from collections.abc import Callable
+from contextlib import ExitStack
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Literal
+
+from pydantic import Field, ValidationError
+
+from offbox import console
+from offbox.config import Model, UserConfig, duration_seconds, load_config
+from offbox.errors import OffboxError, PreconditionError
+from offbox.host import host_operation
+from offbox.host.activity import forget, last_activity, worktree_gone
+from offbox.host.destroy import destroy_host_locked
+from offbox.host.ready import Clock, SystemClock, tcp_probe
+from offbox.host.usage import WallClock, describe, exclusive_usage, holders, utc_now
+from offbox.paths import Paths
+from offbox.providers import validate_name
+from offbox.runner import Runner
+from offbox.ssh import write_private
+from offbox.state import HostState, State, StateStore
+
+logger = logging.getLogger(__name__)
+CREATE_WAIT_SECONDS = 900
+RELEASE_WAIT_SECONDS = 60
+CREATING_GRACE = timedelta(minutes=30)
+REAP_LOCK_WAIT_SECONDS = 5
+Action = Literal["kept", "destroy", "destroyed", "skipped", "failed"]
+
+
+class ReapOutcome(Model):
+    name: str
+    action: Action
+    reason: str
+    lost_environments: list[str] = Field(default_factory=list[str])
+    error: str | None = None
+
+
+class ReapReport(Model):
+    dry_run: bool = False
+    hosts: list[ReapOutcome] = Field(default_factory=list[ReapOutcome])
+
+    @property
+    def exit_code(self) -> int:
+        return 1 if any(outcome.action == "failed" for outcome in self.hosts) else 0
+
+
+class ReapRecord(Model):
+    finished_at: datetime
+    source: Literal["timer", "manual"]
+    report: ReapReport
+
+
+class ReapHistory(Model):
+    timer: ReapRecord | None = None
+    manual: ReapRecord | None = None
+
+
+class Decision(Model):
+    destroy: bool
+    reason: str
+
+
+def _record_path(paths: Paths) -> Path:
+    return paths.state / "reap.json"
+
+
+def read_history(paths: Paths) -> ReapHistory:
+    try:
+        return ReapHistory.model_validate_json(_record_path(paths).read_text())
+    except FileNotFoundError:
+        return ReapHistory()
+    except (OSError, ValidationError):
+        logger.warning("ignoring unreadable reap record", exc_info=True)
+        return ReapHistory()
+
+
+def timer_status(paths: Paths) -> tuple[datetime | None, str | None]:
+    """When the timer last reaped and the first failure it reported, for health checks."""
+    record = read_history(paths).timer
+    if record is None:
+        return None, None
+    failed = next((outcome for outcome in record.report.hosts if outcome.action == "failed"), None)
+    return record.finished_at, None if failed is None else f"{failed.name}: {failed.error}"
+
+
+def lost_environment_hint(paths: Paths, env: str) -> str | None:
+    """How a missing environment record was lost, when the last reap removed it."""
+    history = read_history(paths)
+    records = sorted(
+        (record for record in (history.timer, history.manual) if record is not None), key=lambda item: item.finished_at, reverse=True
+    )
+    for record in records:
+        for outcome in record.report.hosts:
+            if outcome.action == "destroyed" and env in outcome.lost_environments:
+                stamp = record.finished_at.astimezone().strftime("%Y-%m-%d %H:%M")
+                return f"host {outcome.name} was reaped at {stamp} ({outcome.reason}); its Docker data is gone"
+    return None
+
+
+def _format(seconds: float) -> str:
+    minutes = int(seconds // 60)
+    return f"{minutes // 60}h{minutes % 60:02d}m" if minutes >= 60 else f"{minutes}m"
+
+
+def host_activity(paths: Paths, state: State, host: HostState) -> datetime:
+    return max([host.created_at, *(last_activity(paths, env) for env in state.envs.values() if env.host == host.name)])
+
+
+def reap_decision(host: HostState, activity: datetime, now: datetime, *, operation_free: bool, empty: bool = False) -> Decision:
+    """Whether an idle, empty or abandoned auto-managed host should be destroyed now; running commands are guarded by the usage lock."""
+    if not host.auto:
+        return Decision(destroy=False, reason="manual host")
+    if host.phase == "ready" and empty:
+        return Decision(destroy=True, reason="no environments use it")
+    if host.phase != "ready":
+        if not operation_free:
+            return Decision(destroy=False, reason="creation in progress")
+        if now - host.created_at > CREATING_GRACE:
+            return Decision(destroy=True, reason="creation did not finish")
+        return Decision(destroy=False, reason="creation started recently")
+    if host.idle_seconds is None:
+        return Decision(destroy=False, reason="idle reaping is off")
+    idle = (now - activity).total_seconds()
+    if idle > host.idle_seconds:
+        return Decision(destroy=True, reason=f"idle for {_format(idle)}")
+    return Decision(destroy=False, reason=f"last used {_format(max(idle, 0))} ago; reaped after {_format(host.idle_seconds)} idle")
+
+
+def _destroy(paths: Paths, runner: Runner, host: HostState, *, executable: Path | None) -> list[str]:
+    state = StateStore(paths.state).read()
+    lost = sorted(name for name, env in state.envs.items() if env.host == host.name)
+    destroy_host_locked(paths, runner, host.name, executable=executable)
+    for name in lost:
+        forget(paths, name)
+    return lost
+
+
+def ensure_host(
+    paths: Paths,
+    runner: Runner,
+    name: str,
+    *,
+    clock: Clock | None = None,
+    executable: Path | None = None,
+    probe: Callable[[str, int, float], bool] = tcp_probe,
+    ssh_config: Path | None = None,
+    timer_installed: Callable[[Paths], bool] | None = None,
+) -> HostState:
+    """Return the ready host NAME, creating it when its user preset is auto-managed; parallel callers create it once."""
+    validate_name(name)
+    host = StateStore(paths.state).read().hosts.get(name)
+    if host is not None and host.phase == "ready":
+        return host
+    preset = load_config(paths.user_config, UserConfig).hosts.get(name)
+    if host is None and (preset is None or not preset.auto):
+        raise PreconditionError(f"Unknown host: {name}", "Run offbox host list or create this host.")
+    with host_operation(paths, name, wait_timeout=CREATE_WAIT_SECONDS, clock=clock or SystemClock()):
+        host = StateStore(paths.state).read().hosts.get(name)
+        if host is not None and host.phase == "ready":
+            return host
+        if host is not None:
+            raise PreconditionError(f"Host {name} did not finish creating", f"Run offbox host destroy {name}, then retry.")
+        if preset is None or not preset.auto:
+            raise PreconditionError(f"Unknown host: {name}", "Run offbox host list or create this host.")
+        idle = duration_seconds(preset.idle)
+        limit = (
+            f"destroyed after {_format(idle)} idle or when its last environment is removed"
+            if idle
+            else "destroyed when its last environment is removed"
+        )
+        console.warn(f"Creating auto-managed host {name}: billable until destroyed; {limit}")
+        if idle and timer_installed is not None:
+            if not timer_installed(paths):
+                console.info("Idle reaping needs the timer: offbox host reap --install-timer")
+            else:
+                from offbox.host.timer import check_installed_credentials
+
+                problem = check_installed_credentials(paths)
+                if problem is not None:
+                    console.warn(f"The idle reaper cannot destroy {name}: {problem}")
+        from offbox.host.create import CreateOptions, create_host_locked
+
+        return create_host_locked(
+            paths,
+            runner,
+            CreateOptions(name=name, auto=True, idle_seconds=idle),
+            executable=executable,
+            clock=clock,
+            probe=probe,
+            ssh_config=ssh_config,
+        )
+
+
+def release_if_unused(
+    paths: Paths,
+    runner: Runner,
+    name: str,
+    *,
+    clock: Clock | None = None,
+    executable: Path | None = None,
+    timer_installed: Callable[[Paths], bool] | None = None,
+) -> bool:
+    """Destroy an auto-managed host whose last environment is gone; called after rm released the host lock."""
+    store = StateStore(paths.state)
+    host = store.read().hosts.get(name)
+    if host is None or not host.auto:
+        return False
+    if any(env.host == name and not worktree_gone(env.worktree_path) for env in store.read().envs.values()):
+        return False
+    timer = clock or SystemClock()
+    with exclusive_usage(paths, name, wait=RELEASE_WAIT_SECONDS, clock=timer) as free:
+        if not free:
+            fix = f"run offbox host reap {name} once they finish"
+            if timer_installed is not None and not timer_installed(paths):
+                fix += ", or install the idle reaper with offbox host reap --install-timer"
+            console.warn(f"Host {name} is still in use by: {describe(holders(paths, name))}. Fix: {fix}.")
+            return False
+        with host_operation(paths, name, wait_timeout=RELEASE_WAIT_SECONDS, clock=timer):
+            state = store.read()
+            host = state.hosts.get(name)
+            if host is None or not host.auto:
+                return False
+            if any(env.host == name and not worktree_gone(env.worktree_path) for env in state.envs.values()):
+                return False
+            console.step(f"Destroying auto-managed host {name}: its last environment was removed")
+            lost = _destroy(paths, runner, host, executable=executable)
+            previous = read_history(paths).manual
+            earlier = [outcome for outcome in previous.report.hosts if outcome.name != name] if previous is not None else []
+            outcome = ReapOutcome(name=name, action="destroyed", reason="last environment removed", lost_environments=lost)
+            _record(paths, ReapReport(hosts=[*earlier, outcome]), "manual", utc_now())
+            return True
+
+
+def reap(
+    paths: Paths,
+    runner: Runner,
+    name: str | None = None,
+    *,
+    dry_run: bool = False,
+    source: Literal["timer", "manual"] = "manual",
+    clock: Clock | None = None,
+    now: WallClock = utc_now,
+    executable: Path | None = None,
+) -> ReapReport:
+    store = StateStore(paths.state)
+    state = store.read()
+    if name is not None:
+        validate_name(name)
+        if name not in state.hosts:
+            raise PreconditionError(f"Unknown host: {name}", "Run offbox host list.")
+        names = [name]
+    else:
+        names = sorted(host.name for host in state.hosts.values() if host.auto)
+    report = ReapReport(dry_run=dry_run)
+    timer = clock or SystemClock()
+    for current in names:
+        try:
+            report.hosts.append(_reap_one(paths, runner, current, dry_run=dry_run, clock=timer, now=now, executable=executable))
+        except OffboxError as exc:
+            logger.exception("reaping %s failed", current)
+            report.hosts.append(ReapOutcome(name=current, action="failed", reason="destroy failed", error=f"{exc.message} Fix: {exc.fix}"))
+    if not dry_run:
+        _record(paths, report, source, now())
+    return report
+
+
+def _record(paths: Paths, report: ReapReport, source: Literal["timer", "manual"], finished_at: datetime) -> None:
+    history = read_history(paths).model_copy(update={source: ReapRecord(finished_at=finished_at, source=source, report=report)})
+    write_private(_record_path(paths), history.model_dump_json(indent=2) + "\n")
+
+
+def clear_lost(paths: Paths, env: str) -> None:
+    """Forget that ENV was lost to a reap once it is registered again or removed, so later hints stay true."""
+    history = read_history(paths)
+    changed = False
+    for record in (history.timer, history.manual):
+        if record is None:
+            continue
+        for outcome in record.report.hosts:
+            if env in outcome.lost_environments:
+                outcome.lost_environments.remove(env)
+                changed = True
+    if changed:
+        write_private(_record_path(paths), history.model_dump_json(indent=2) + "\n")
+
+
+def _reap_one(
+    paths: Paths, runner: Runner, name: str, *, dry_run: bool, clock: Clock, now: WallClock, executable: Path | None
+) -> ReapOutcome:
+    store = StateStore(paths.state)
+    host = store.read().hosts.get(name)
+    if host is None:
+        return ReapOutcome(name=name, action="skipped", reason="host no longer registered")
+    if not host.auto:
+        return ReapOutcome(name=name, action="skipped", reason="manual host")
+    with ExitStack() as stack:
+        if not stack.enter_context(exclusive_usage(paths, name, clock=clock)):
+            return ReapOutcome(name=name, action="skipped", reason=f"in use by {describe(holders(paths, name))}")
+        try:
+            stack.enter_context(host_operation(paths, name, wait_timeout=REAP_LOCK_WAIT_SECONDS, clock=clock))
+            operation_free = True
+        except PreconditionError:
+            operation_free = False
+        state = store.read()
+        host = state.hosts.get(name)
+        if host is None:
+            return ReapOutcome(name=name, action="skipped", reason="host no longer registered")
+        if not operation_free and host.phase == "ready":
+            return ReapOutcome(name=name, action="skipped", reason="another host operation is running")
+        empty = not any(env.host == name and not worktree_gone(env.worktree_path) for env in state.envs.values())
+        decision = reap_decision(host, host_activity(paths, state, host), now(), operation_free=operation_free, empty=empty)
+        lost = sorted(env.name for env in state.envs.values() if env.host == name)
+        if not decision.destroy:
+            return ReapOutcome(name=name, action="kept", reason=decision.reason)
+        if dry_run:
+            return ReapOutcome(name=name, action="destroy", reason=decision.reason, lost_environments=lost)
+        console.step(f"Reaping auto-managed host {name}: {decision.reason}")
+        lost = _destroy(paths, runner, host, executable=executable)
+        return ReapOutcome(name=name, action="destroyed", reason=decision.reason, lost_environments=lost)
+    return ReapOutcome(name=name, action="skipped", reason="unreachable")

offbox/host/cloudinit.py

@@ -0,0 +1,44 @@
+from importlib.resources import files
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from offbox import DEFAULT_IMAGE, __version__
+from offbox.errors import PreconditionError
+from offbox.logging import register_secrets
+from offbox.runner import Runner
+
+
+def bootstrap_script() -> str:
+    return files("offbox").joinpath("_data", "host", "host-init.sh").read_text(encoding="utf-8")
+
+
+def render_cloud_init(public_key: str, private_host_key: str, public_host_key: str, *, script: str | None = None) -> str:
+    template = files("offbox").joinpath("_data", "host", "cloud-init.yaml.j2").read_text(encoding="utf-8")
+    register_secrets({"OFFBOX_HOST_PRIVATE_KEY": private_host_key})
+    return (
+        Environment(undefined=StrictUndefined, keep_trailing_newline=True)
+        .from_string(template)
+        .render(
+            private_host_key=private_host_key,
+            public_host_key=public_host_key,
+            script=bootstrap_script() if script is None else script,
+            bootstrap_argv=[
+                "env",
+                "OFFBOX_HARDEN=1",
+                f"OFFBOX_VERSION={__version__}",
+                "/usr/local/sbin/offbox-host-init",
+                DEFAULT_IMAGE,
+                public_key,
+            ],
+        )
+    )
+
+
+def generate_host_key(runner: Runner, path: Path) -> tuple[str, str]:
+    runner.run(["ssh-keygen", "-t", "ed25519", "-N", "", "-C", "offbox-host", "-f", str(path)])
+    try:
+        path.chmod(0o600)
+        return path.read_text(encoding="utf-8"), path.with_name(path.name + ".pub").read_text(encoding="utf-8").strip()
+    except (OSError, UnicodeError) as exc:
+        raise PreconditionError("Cannot read generated SSH host key", "Check state directory permissions and ssh-keygen.") from exc