offbox 0.1.0__py3-none-any.whl

offbox/host/timer.py

@@ -0,0 +1,227 @@
+import os
+import platform
+import re
+import shutil
+import sys
+import tempfile
+from collections.abc import Mapping
+from datetime import datetime, timedelta
+from html import escape as _escape
+from html import unescape
+from pathlib import Path
+
+from offbox import console
+from offbox.config import UserConfig, load_config
+from offbox.errors import PreconditionError
+from offbox.paths import Paths
+from offbox.providers import credentials_env, discover_provider
+from offbox.runner import Runner
+from offbox.ssh import write_private
+
+INTERVAL_SECONDS = 900
+
+
+def escape(value: str) -> str:
+    return _escape(value, quote=False)
+
+
+STALE_AFTER = timedelta(minutes=60)
+LAUNCHD_LABEL = "dev.offbox.reap"
+SYSTEMD_UNIT = "offbox-reap"
+_PASSED_THROUGH = ("XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME", "GOOGLE_PROJECT")
+_TOOLS = ("tofu", "terraform", "ssh", "git", "mutagen")
+
+
+def unit_files(paths: Paths, home: Path, system: str | None = None) -> list[Path]:
+    """Scheduler files: a launchd agent under HOME on macOS, systemd user units beside offbox's XDG config elsewhere."""
+    if (system or platform.system()) == "Darwin":
+        return [home / "Library/LaunchAgents" / f"{LAUNCHD_LABEL}.plist"]
+    base = paths.config.parent / "systemd/user"
+    return [base / f"{SYSTEMD_UNIT}.service", base / f"{SYSTEMD_UNIT}.timer"]
+
+
+def installed(paths: Paths, *, home: Path | None = None, system: str | None = None) -> bool:
+    return all(path.is_file() for path in unit_files(paths, home or Path.home(), system))
+
+
+def executable_path() -> Path:
+    """The installed offbox command the timer runs; ephemeral uvx or temporary environments are refused."""
+    found = shutil.which("offbox")
+    candidate = Path(found).resolve() if found else Path(sys.argv[0]).resolve()
+    text = str(candidate)
+    temporary = Path(tempfile.gettempdir()).resolve()
+    if (
+        not candidate.is_file()
+        or candidate.is_relative_to(temporary)
+        or "/.cache/uv/" in text
+        or "/uv/cache/" in text
+        or text.startswith("/tmp/")
+    ):
+        raise PreconditionError(
+            f"The idle reaper needs an installed offbox, not {candidate}", "Install it with uv tool install offbox, then retry."
+        )
+    return candidate
+
+
+def unit_environment(paths: Paths, environ: Mapping[str, str], home: Path) -> dict[str, str]:
+    """The environment a scheduled reap runs with: tool directories and settings locations, never credentials."""
+    directories: list[str] = []
+    for tool in _TOOLS:
+        found = shutil.which(tool, path=environ.get("PATH"))
+        if found:
+            directories.append(str(Path(found).parent))
+    from offbox.tools import cached_mutagen
+
+    mutagen = cached_mutagen(paths.cache)
+    if mutagen is not None:
+        directories.append(str(mutagen.parent))
+    directories.extend(["/usr/bin", "/bin", "/usr/sbin", "/sbin"])
+    environment = {"HOME": str(home), "PATH": ":".join(dict.fromkeys(directories))}
+    environment.update({name: environ[name] for name in _PASSED_THROUGH if environ.get(name)})
+    return environment
+
+
+def preflight(paths: Paths, environment: Mapping[str, str], home: Path) -> None:
+    """Fail unless every auto-managed preset's credentials are readable without the current shell's environment."""
+    for name, preset in sorted(load_config(paths.user_config, UserConfig).hosts.items()):
+        if not preset.auto:
+            continue
+        provider = discover_provider(preset.provider, paths)
+        try:
+            credentials_env(provider.manifest, paths, environment)
+        except PreconditionError as exc:
+            fix = (
+                f"Store the Hetzner token in {paths.credentials} (mode 0600) instead of only exporting HCLOUD_TOKEN."
+                if provider.manifest.name == "hetzner"
+                else f"Export them in the shell that runs --install-timer, which copies non-secret provider variables into the timer: {exc.fix}"
+            )
+            raise PreconditionError(f"The idle reaper cannot read credentials for host {name}: {exc.message}", fix) from exc
+        if provider.manifest.name == "gcp" and not (home / ".config/gcloud/application_default_credentials.json").is_file():
+            raise PreconditionError(
+                f"The idle reaper cannot find Google application default credentials for host {name}",
+                "Run gcloud auth application-default login, then retry.",
+            )
+
+
+def render_launchd(executable: Path, environment: Mapping[str, str], log: Path) -> str:
+    variables = "".join(f"<key>{escape(key)}</key><string>{escape(value)}</string>" for key, value in sorted(environment.items()))
+    return (
+        '<?xml version="1.0" encoding="UTF-8"?>\n'
+        '<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n'
+        '<plist version="1.0"><dict>'
+        f"<key>Label</key><string>{LAUNCHD_LABEL}</string>"
+        f"<key>ProgramArguments</key><array><string>{escape(str(executable))}</string><string>host</string>"
+        "<string>reap</string><string>--from-timer</string></array>"
+        f"<key>EnvironmentVariables</key><dict>{variables}</dict>"
+        f"<key>StartInterval</key><integer>{INTERVAL_SECONDS}</integer>"
+        "<key>RunAtLoad</key><true/>"
+        f"<key>StandardOutPath</key><string>{escape(str(log))}</string>"
+        f"<key>StandardErrorPath</key><string>{escape(str(log))}</string>"
+        "</dict></plist>\n"
+    )
+
+
+def _quote(value: str) -> str:
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"').replace("%", "%%").replace("$", "$$")
+    return '"' + escaped + '"'
+
+
+def render_systemd(executable: Path, environment: Mapping[str, str]) -> tuple[str, str]:
+    variables = "".join(f"Environment={_quote(f'{key}={value}')}\n" for key, value in sorted(environment.items()))
+    service = (
+        "[Unit]\nDescription=Destroy idle offbox hosts\n\n"
+        f"[Service]\nType=oneshot\n{variables}ExecStart={_quote(str(executable))} host reap --from-timer\n"
+    )
+    timer = "[Unit]\nDescription=Run the offbox idle reaper every 15 minutes\n\n[Timer]\nOnCalendar=*:0/15\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n"
+    return service, timer
+
+
+def install(
+    paths: Paths, runner: Runner, *, home: Path | None = None, system: str | None = None, environ: Mapping[str, str] | None = None
+) -> list[Path]:
+    root = home or Path.home()
+    current = system or platform.system()
+    executable = executable_path()
+    environment = unit_environment(paths, os.environ if environ is None else environ, root)
+    preflight(paths, environment, root)
+    files = unit_files(paths, root, current)
+    if current == "Darwin":
+        write_private(files[0], render_launchd(executable, environment, paths.logs / "reaper.log"))
+        domain = f"gui/{os.getuid()}"
+        runner.run(["launchctl", "bootout", domain, str(files[0])], check=False, timeout=30)
+        runner.run(["launchctl", "bootstrap", domain, str(files[0])], timeout=30)
+    elif current == "Linux":
+        service, timer = render_systemd(executable, environment)
+        write_private(files[0], service)
+        write_private(files[1], timer)
+        runner.run(["systemctl", "--user", "daemon-reload"], timeout=30)
+        runner.run(["systemctl", "--user", "enable", "--now", f"{SYSTEMD_UNIT}.timer"], timeout=30)
+    else:
+        raise PreconditionError(f"The idle reaper timer is not supported on {current}", "Run offbox host reap from your own scheduler.")
+    console.success(f"Idle reaper installed: offbox host reap runs every {INTERVAL_SECONDS // 60} minutes")
+    return files
+
+
+def remove(paths: Paths, runner: Runner, *, home: Path | None = None, system: str | None = None) -> bool:
+    root = home or Path.home()
+    current = system or platform.system()
+    files = unit_files(paths, root, current)
+    if not any(path.exists() for path in files):
+        return False
+    if current == "Darwin":
+        runner.run(["launchctl", "bootout", f"gui/{os.getuid()}", str(files[0])], check=False, timeout=30)
+    elif current == "Linux":
+        runner.run(["systemctl", "--user", "disable", "--now", f"{SYSTEMD_UNIT}.timer"], check=False, timeout=30)
+    for path in files:
+        path.unlink(missing_ok=True)
+    if current == "Linux":
+        runner.run(["systemctl", "--user", "daemon-reload"], check=False, timeout=30)
+    return True
+
+
+def health(paths: Paths, now: datetime, last_run: datetime | None, failure: str | None, *, home: Path | None = None) -> str | None:
+    """A warning when the installed timer has stopped running or its last run failed."""
+    files = unit_files(paths, home or Path.home())
+    if not all(path.is_file() for path in files):
+        return None
+    if failure:
+        return f"The last scheduled reap failed: {failure}"
+    reference = last_run
+    if reference is None:
+        try:
+            reference = datetime.fromtimestamp(files[0].stat().st_mtime, now.tzinfo)
+        except OSError:
+            return None
+    if now - reference > STALE_AFTER:
+        return "The idle reaper has not run for over an hour (a sleeping laptop also delays it). Fix: offbox doctor, or reinstall with offbox host reap --install-timer."
+    return None
+
+
+def unit_problem(paths: Paths, *, home: Path | None = None, system: str | None = None) -> str | None:
+    """A problem with an installed timer's target executable, such as an uninstalled offbox."""
+    files = unit_files(paths, home or Path.home(), system)
+    if not all(path.is_file() for path in files):
+        return None
+    try:
+        text = files[0].read_text()
+    except OSError:
+        return "The idle reaper unit cannot be read"
+    found = re.search(r"<key>ProgramArguments</key><array><string>([^<]+)</string>", text) or re.search(
+        r'^ExecStart="([^"]+)"', text, re.MULTILINE
+    )
+    if found is None:
+        return "The idle reaper unit has no executable"
+    executable = Path(unescape(found[1]).replace("%%", "%").replace("$$", "$"))
+    if not executable.is_file():
+        return f"The idle reaper runs {executable}, which no longer exists"
+    return None
+
+
+def check_installed_credentials(paths: Paths, *, home: Path | None = None) -> str | None:
+    """Re-run the timer's credential preflight; None when every auto-managed preset can be destroyed by the timer."""
+    root = home or Path.home()
+    try:
+        preflight(paths, unit_environment(paths, os.environ, root), root)
+    except PreconditionError as exc:
+        return f"{exc.message}. Fix: {exc.fix}"
+    return None

offbox/host/usage.py

@@ -0,0 +1,123 @@
+import logging
+import os
+import uuid
+from collections.abc import Callable, Generator
+from contextlib import contextmanager
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from offbox.config import Model
+from offbox.errors import PreconditionError
+from offbox.host.ready import Clock, SystemClock
+from offbox.locking import acquire_lock
+from offbox.paths import Paths
+from offbox.providers import validate_name
+
+logger = logging.getLogger(__name__)
+USAGE_WAIT_SECONDS = 600
+type WallClock = Callable[[], datetime]
+
+
+def utc_now() -> datetime:
+    return datetime.now(UTC)
+
+
+class UsageHolder(Model):
+    pid: int
+    command: str
+    started_at: datetime
+
+
+def _locations(paths: Paths, name: str) -> tuple[Path, Path]:
+    validate_name(name)
+    directory = paths.state / "host-locks"
+    return directory / f"{name}.usage", directory / f"{name}.users"
+
+
+def _open(path: Path) -> int:
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        return os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+    except OSError as exc:
+        raise PreconditionError("Cannot open host usage lock", "Check state directory permissions.") from exc
+
+
+def _alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
+def holders(paths: Paths, name: str) -> list[UsageHolder]:
+    """Live commands that hold the shared usage lock, pruning entries of processes that have exited."""
+    _, directory = _locations(paths, name)
+    found: list[UsageHolder] = []
+    try:
+        entries = sorted(directory.glob("*.json"))
+    except OSError:
+        return found
+    for entry in entries:
+        try:
+            holder = UsageHolder.model_validate_json(entry.read_text())
+        except (OSError, ValidationError):
+            logger.info("ignoring unreadable usage holder %s", entry)
+            continue
+        if _alive(holder.pid):
+            found.append(holder)
+        else:
+            entry.unlink(missing_ok=True)
+    return found
+
+
+def describe(found: list[UsageHolder]) -> str:
+    return ", ".join(f"{holder.command} (pid {holder.pid})" for holder in found) or "an offbox command"
+
+
+@contextmanager
+def shared_usage(
+    paths: Paths, name: str, command: str, *, wait: float = USAGE_WAIT_SECONDS, clock: Clock | None = None, now: WallClock = utc_now
+) -> Generator[None]:
+    """Mark the host as in use for the duration of the block; destroy paths skip a host while any holder remains."""
+    lock, directory = _locations(paths, name)
+    descriptor = _open(lock)
+    entry = directory / f"{os.getpid()}-{uuid.uuid4().hex}.json"
+    try:
+        acquire_lock(
+            descriptor,
+            timeout=wait,
+            clock=clock or SystemClock(),
+            message=f"Host {name} is being destroyed",
+            fix="Retry once the destroy finishes; offbox up recreates an auto-managed host.",
+            shared=True,
+        )
+        try:
+            directory.mkdir(parents=True, exist_ok=True, mode=0o700)
+            entry.write_text(UsageHolder(pid=os.getpid(), command=command, started_at=now()).model_dump_json())
+        except OSError:
+            logger.warning("cannot record usage holder for %s", name, exc_info=True)
+        yield
+    finally:
+        entry.unlink(missing_ok=True)
+        os.close(descriptor)
+
+
+@contextmanager
+def exclusive_usage(paths: Paths, name: str, *, wait: float = 0, clock: Clock | None = None) -> Generator[bool]:
+    """Yield True with the host reserved for destruction, or False when commands still use it after waiting."""
+    lock, _ = _locations(paths, name)
+    descriptor = _open(lock)
+    try:
+        try:
+            acquire_lock(descriptor, timeout=wait, clock=clock or SystemClock(), message="busy", fix="busy")
+        except PreconditionError:
+            yield False
+            return
+        yield True
+    finally:
+        os.close(descriptor)

offbox/host/warnings.py

@@ -0,0 +1,61 @@
+import logging
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pydantic import Field, ValidationError
+
+from offbox.config import Model
+from offbox.paths import Paths
+from offbox.ssh import write_private
+
+logger = logging.getLogger(__name__)
+
+
+class HostWarning(Model):
+    host: str
+    message: str
+    created_at: datetime
+
+
+class HostWarnings(Model):
+    warnings: list[HostWarning] = Field(default_factory=list[HostWarning])
+
+
+def _warnings_path(paths: Paths) -> Path:
+    return paths.state / "warnings.json"
+
+
+def read_warnings(paths: Paths) -> list[HostWarning]:
+    try:
+        return HostWarnings.model_validate_json(_warnings_path(paths).read_text()).warnings
+    except FileNotFoundError:
+        return []
+    except (OSError, ValidationError):
+        logger.warning("ignoring unreadable host warnings", exc_info=True)
+        return []
+
+
+def _write_warnings(paths: Paths, warnings: list[HostWarning]) -> None:
+    write_private(_warnings_path(paths), HostWarnings(warnings=warnings).model_dump_json(indent=2) + "\n")
+
+
+def dismiss_warning(paths: Paths, name: str) -> bool:
+    current = read_warnings(paths)
+    kept = [warning for warning in current if warning.host != name]
+    if len(kept) == len(current):
+        return False
+    _write_warnings(paths, kept)
+    return True
+
+
+def warn_interrupted(paths: Paths, name: str) -> None:
+    """Keep a reminder after destroying a host whose creation was killed mid-apply, which can leave untracked resources."""
+    warnings = [warning for warning in read_warnings(paths) if warning.host != name]
+    warnings.append(
+        HostWarning(
+            host=name,
+            message=f"Creation of {name} was interrupted; check the provider console for resources labelled offbox-host={name}",
+            created_at=datetime.now(UTC),
+        )
+    )
+    _write_warnings(paths, warnings)

offbox/locking.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import fcntl
+from typing import TYPE_CHECKING
+
+from offbox.errors import PreconditionError
+
+if TYPE_CHECKING:
+    from offbox.host.ready import Clock
+
+
+def acquire_lock(descriptor: int, *, timeout: float = 0, clock: Clock, message: str, fix: str, shared: bool = False) -> None:
+    timer = clock
+    deadline: float | None = None
+    while True:
+        try:
+            fcntl.flock(descriptor, (fcntl.LOCK_SH if shared else fcntl.LOCK_EX) | fcntl.LOCK_NB)
+            return
+        except BlockingIOError as exc:
+            if timeout <= 0:
+                raise PreconditionError(message, fix) from exc
+            if deadline is None:
+                deadline = timer.now() + timeout
+            remaining = deadline - timer.now()
+            if remaining <= 0:
+                raise PreconditionError(f"{message} after waiting {timeout:g}s", fix) from exc
+            timer.sleep(min(0.05, remaining))

offbox/logging.py

@@ -0,0 +1,154 @@
+import json
+import logging
+import os
+import re
+import uuid
+from collections.abc import Mapping, Sequence
+from contextvars import ContextVar
+from datetime import UTC, date, datetime, timedelta
+from pathlib import Path
+
+from offbox.config import Model
+from offbox.paths import Paths
+
+_registered_secrets: set[str] = set()
+_SECRET_NAME = re.compile(r"(?:TOKEN|SECRET|PASSWORD|KEY|CREDENTIALS|KEYFILE_JSON)$", re.IGNORECASE)
+_STANDALONE_VALUE_LENGTH = 20
+_NO_LOG_OPTIONS = frozenset({"--help", "--version"})
+_invocation: ContextVar[str] = ContextVar("offbox_invocation", default="")
+_environment: ContextVar[str | None] = ContextVar("offbox_environment", default=None)
+
+
+class LogEntry(Model):
+    timestamp: datetime
+    invocation: str
+    env: str | None
+    level: str
+    logger: str
+    message: str
+
+
+def current_invocation() -> str:
+    return _invocation.get()
+
+
+def bind_environment(name: str) -> None:
+    _environment.set(name)
+    logging.getLogger(__name__).debug("worktree context established")
+
+
+def close_logs() -> None:
+    root = logging.getLogger("offbox")
+    for handler in root.handlers[:]:
+        root.removeHandler(handler)
+        handler.close()
+
+
+class StructuredFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        message = record.getMessage()
+        if record.exc_info:
+            message += "\n" + self.formatException(record.exc_info)
+        return LogEntry(
+            timestamp=datetime.fromtimestamp(record.created, UTC),
+            invocation=_invocation.get(),
+            env=_environment.get(),
+            level=record.levelname,
+            logger=record.name,
+            message=redact(message),
+        ).model_dump_json()
+
+
+def register_secrets(env: Mapping[str, str]) -> None:
+    values = [value for name, value in env.items() if value and _SECRET_NAME.search(name)]
+    _registered_secrets.update(values)
+    _registered_secrets.update(part for value in values for part in value.splitlines() if part)
+
+
+def register_env_values(env: Mapping[str, str]) -> None:
+    """Redact forwarded values as NAME=value; on their own only for secret-looking names or long values, so paths stay readable."""
+    for name, value in env.items():
+        if not value:
+            continue
+        named = _SECRET_NAME.search(name) is not None
+        standalone = named or len(value) >= _STANDALONE_VALUE_LENGTH
+        _registered_secrets.add(f"{name}={value}")
+        if standalone:
+            _registered_secrets.add(value)
+        _registered_secrets.update(part for part in value.splitlines() if part and (named or len(part) >= _STANDALONE_VALUE_LENGTH))
+
+
+def register_exec_secrets(argv: Sequence[str]) -> None:
+    command = next((index for index, value in enumerate(argv) if not value.startswith("-")), None)
+    if command is None or argv[command] != "exec":
+        return
+    index = command + 1
+    while index < len(argv):
+        argument = argv[index]
+        if argument == "--":
+            return
+        entry: str | None = None
+        if argument == "--env":
+            index += 1
+            if index >= len(argv) or argv[index] == "--":
+                return
+            entry = argv[index]
+        elif argument.startswith("--env="):
+            entry = argument[len("--env=") :]
+        if entry is not None:
+            name, separator, value = entry.partition("=")
+            if separator:
+                register_env_values({name: value})
+        index += 1
+
+
+def redact(value: str, env: Mapping[str, str] | None = None) -> str:
+    values = os.environ if env is None else env
+    secrets = sorted(
+        _registered_secrets | {item for name, item in values.items() if item and _SECRET_NAME.search(name)}, key=len, reverse=True
+    )
+    for secret in secrets:
+        for variant in sorted({secret, repr(secret)[1:-1], json.dumps(secret)[1:-1]}, key=len, reverse=True):
+            value = value.replace(variant, "[REDACTED]")
+    return value
+
+
+class RedactingFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        return redact(super().format(record))
+
+
+def configure(paths: Paths, argv: Sequence[str], day: date | None = None) -> Path:
+    current = day or date.today()
+    paths.logs.mkdir(parents=True, exist_ok=True, mode=0o700)
+    unpruned: list[Path] = []
+    for candidate in paths.logs.glob("*.log"):
+        try:
+            file_day = date.fromisoformat(candidate.stem)
+        except ValueError:
+            continue
+        if file_day <= current - timedelta(days=14):
+            try:
+                candidate.unlink(missing_ok=True)
+            except OSError:
+                unpruned.append(candidate)
+    target = paths.log_file(current)
+    handler = logging.FileHandler(target, encoding="utf-8")
+    target.chmod(0o600)
+    handler.setFormatter(StructuredFormatter())
+    root = logging.getLogger("offbox")
+    close_logs()
+    _invocation.set(uuid.uuid4().hex)
+    _environment.set(None)
+    root.setLevel(logging.DEBUG)
+    root.addHandler(handler)
+    root.propagate = False
+    for candidate in unpruned:
+        root.warning("cannot prune old log %s", candidate)
+    if not _NO_LOG_OPTIONS.intersection(_options(argv)):
+        root.info("invocation argv=%s", [redact(arg) for arg in argv])
+    return target
+
+
+def _options(argv: Sequence[str]) -> list[str]:
+    return list(argv[: argv.index("--")] if "--" in argv else argv)