offbox 0.1.0__py3-none-any.whl

offbox/project.py

@@ -0,0 +1,187 @@
+import os
+import tomllib
+import types
+import typing
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
+
+from offbox.config import Model, ProjectConfig, UserConfig, load_config
+from offbox.errors import PreconditionError
+from offbox.paths import Paths
+
+PROJECT_FILE = "offbox.toml"
+LOCAL_FILE = "offbox.local.toml"
+Source = Literal["default", "user", "project", "local", "env"]
+type Layer = dict[str, dict[str, JsonValue]]
+_JSON = TypeAdapter[JsonValue](JsonValue)
+_LAYER = TypeAdapter[Layer](dict[str, dict[str, JsonValue]])
+_EXCLUSIVE = {"env.image": ("env.dockerfile",), "env.dockerfile": ("env.image",)}
+
+
+class SettingKey(Model):
+    section: str
+    field: str
+    is_list: bool
+    textual: bool
+
+
+class Setting(Model):
+    key: str
+    value: JsonValue
+    source: Source
+    env: str
+
+
+class ConfigFiles(Model):
+    user: Path
+    project: Path
+    local: Path
+
+
+class ConfigReport(Model):
+    files: ConfigFiles
+    settings: list[Setting]
+
+
+class ResolvedProject(Model):
+    config: ProjectConfig
+    sources: dict[str, Source]
+
+
+def setting_keys() -> dict[str, SettingKey]:
+    """Every project setting as section.key, with its TOML section and field name."""
+    keys: dict[str, SettingKey] = {}
+    for section, info in ProjectConfig.model_fields.items():
+        model = info.annotation
+        if not (isinstance(model, type) and issubclass(model, BaseModel)):
+            raise PreconditionError(f"Invalid project section {section}", "Report this offbox bug.")
+        for name, field in model.model_fields.items():
+            key = field.alias or name
+            is_list = typing.get_origin(field.annotation) is list
+            scalar = typing.get_args(field.annotation)[0] if is_list else field.annotation
+            members = typing.get_args(scalar) if typing.get_origin(scalar) in (types.UnionType, typing.Union) else (scalar,)
+            textual = any(member in (str, Path) for member in members)
+            keys[f"{section}.{key}"] = SettingKey(section=section, field=key, is_list=is_list, textual=textual)
+    return keys
+
+
+def env_name(key: str) -> str:
+    return "OFFBOX_" + key.upper().replace(".", "_")
+
+
+def parse_value(raw: str) -> JsonValue:
+    """A TOML value such as true, 2, "8g" or [".env"]; text that is not a TOML value is taken as a plain string."""
+    try:
+        value: object = tomllib.loads(f"value = {raw}")["value"]
+        return _JSON.validate_python(value, strict=True)
+    except (tomllib.TOMLDecodeError, ValidationError):
+        return raw
+
+
+def _locations(exc: ValidationError) -> str:
+    return ", ".join(".".join(str(part) for part in error["loc"]) or "<root>" for error in exc.errors(include_input=False))
+
+
+def _validate(layer: Layer, origin: str) -> None:
+    try:
+        ProjectConfig.model_validate(layer)
+    except ValidationError as exc:
+        raise PreconditionError(f"Invalid configuration in {origin}: {_locations(exc)}", f"Correct these keys in {origin}.") from exc
+
+
+def read_layer(path: Path) -> Layer:
+    try:
+        with path.open("rb") as source:
+            data: object = tomllib.load(source)
+    except FileNotFoundError:
+        return {}
+    except (tomllib.TOMLDecodeError, UnicodeError, OSError) as exc:
+        raise PreconditionError(f"Cannot read configuration {path}", f"Fix the TOML syntax and permissions in {path}.") from exc
+    try:
+        layer = _LAYER.validate_python(data, strict=True)
+    except ValidationError as exc:
+        raise PreconditionError(
+            f"Invalid configuration in {path}: {_locations(exc)}", f"Use [section] tables with plain TOML values in {path}."
+        ) from exc
+    _validate(layer, str(path))
+    return layer
+
+
+def env_layer(environ: Mapping[str, str]) -> Layer:
+    keys = setting_keys()
+    names = {env_name(key): setting for key, setting in keys.items()}
+    prefixes = {f"OFFBOX_{setting.section.upper()}_" for setting in keys.values()}
+    layer: Layer = {}
+    for name, raw in sorted(environ.items()):
+        setting = names.get(name)
+        if setting is not None:
+            value: JsonValue = parse_value(raw)
+            if setting.is_list and not raw.strip():
+                value = []
+            elif setting.is_list and not isinstance(value, list):
+                value = [raw if setting.textual else value]
+            elif not setting.is_list and setting.textual:
+                value = raw
+            layer.setdefault(setting.section, {})[setting.field] = value
+        elif any(name.startswith(prefix) for prefix in prefixes):
+            raise PreconditionError(
+                f"Unknown setting {name}",
+                f"Unset {name}; setting variables are OFFBOX_<SECTION>_<KEY> for keys in the configuration reference.",
+            )
+    _validate(layer, "OFFBOX_* environment variables")
+    return layer
+
+
+def user_layer(paths: Paths) -> Layer:
+    project = load_config(paths.user_config, UserConfig).project
+    _validate(project, f"{paths.user_config} [project]")
+    return project
+
+
+def resolve(paths: Paths, root: Path, environ: Mapping[str, str] | None = None) -> ResolvedProject:
+    """Merge defaults, user [project] defaults, offbox.toml, offbox.local.toml and OFFBOX_* variables; later sources win per key."""
+    layers: list[tuple[Source, Layer]] = [
+        ("user", user_layer(paths)),
+        ("project", read_layer(root / PROJECT_FILE)),
+        ("local", read_layer(root / LOCAL_FILE)),
+        ("env", env_layer(os.environ if environ is None else environ)),
+    ]
+    merged: Layer = {}
+    sources: dict[str, Source] = dict.fromkeys(setting_keys(), "default")
+    for source, layer in layers:
+        for section, values in layer.items():
+            for key, value in values.items():
+                current = merged.setdefault(section, {})
+                for paired in _EXCLUSIVE.get(f"{section}.{key}", ()):
+                    current.pop(paired.split(".", 1)[1], None)
+                    sources[paired] = "default"
+                current[key] = value
+                sources[f"{section}.{key}"] = source
+    try:
+        config = ProjectConfig.model_validate(merged)
+    except ValidationError as exc:
+        details = "; ".join(error["msg"] for error in exc.errors(include_input=False))
+        raise PreconditionError(
+            f"Merged configuration is invalid: {details}",
+            "Correct the named settings in config.toml, offbox.toml, offbox.local.toml or OFFBOX_* variables.",
+        ) from exc
+    return ResolvedProject(config=config, sources=sources)
+
+
+def load_project(paths: Paths, root: Path, environ: Mapping[str, str] | None = None) -> ProjectConfig:
+    return resolve(paths, root, environ).config
+
+
+def report(paths: Paths, root: Path, environ: Mapping[str, str] | None = None) -> ConfigReport:
+    resolved = resolve(paths, root, environ)
+    dumped = resolved.config.model_dump(mode="json", by_alias=True)
+    settings = [
+        Setting(
+            key=key, value=_JSON.validate_python(dumped[setting.section][setting.field]), source=resolved.sources[key], env=env_name(key)
+        )
+        for key, setting in setting_keys().items()
+    ]
+    return ConfigReport(files=ConfigFiles(user=paths.user_config, project=root / PROJECT_FILE, local=root / LOCAL_FILE), settings=settings)

offbox/providers.py

@@ -0,0 +1,117 @@
+import hashlib
+import os
+import re
+import shutil
+import tomllib
+from collections.abc import Mapping
+from importlib.resources import files
+from pathlib import Path
+from typing import Literal
+
+from pydantic import Field, ValidationError
+
+from offbox.config import Model, hetzner_token
+from offbox.errors import PreconditionError
+from offbox.logging import register_secrets
+from offbox.paths import Paths
+
+
+class ProviderDefaults(Model):
+    size: str | None = None
+    region: str | None = None
+    image: str | None = None
+
+
+class ProviderManifest(Model):
+    name: str
+    description: str
+    driver: Literal["tofu", "none"]
+    credentials_env: list[str] = Field(default_factory=list[str])
+    defaults: ProviderDefaults = Field(default_factory=ProviderDefaults)
+
+
+class Provider(Model):
+    manifest: ProviderManifest
+    directory: Path
+
+
+def validate_name(value: str) -> str:
+    if re.fullmatch(r"[a-z][a-z0-9-]{0,31}", value) is None:
+        raise PreconditionError(f"Invalid name: {value!r}", "Use 1–32 lowercase letters, digits or hyphens, starting with a letter.")
+    return value
+
+
+def load_provider(directory: Path, name: str | None = None) -> Provider:
+    path = directory / "provider.toml"
+    try:
+        with path.open("rb") as source:
+            manifest = ProviderManifest.model_validate(tomllib.load(source))
+        validate_name(manifest.name)
+        if name is not None and manifest.name != name:
+            raise PreconditionError(f"Provider name does not match {name}: {path}", "Correct the provider manifest name.")
+        for variable in manifest.credentials_env:
+            if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", variable) is None:
+                raise PreconditionError(f"Invalid credential variable in {path}", "Use valid environment variable names.")
+        return Provider(manifest=manifest, directory=directory)
+    except (OSError, UnicodeError, tomllib.TOMLDecodeError, ValidationError) as exc:
+        raise PreconditionError(f"Cannot load provider {path}", "Check the provider manifest syntax, fields and permissions.") from exc
+
+
+def discover_provider(name: str, paths: Paths) -> Provider:
+    validate_name(name)
+    custom = paths.config / "providers" / name
+    if custom.exists():
+        return load_provider(custom, name)
+    resource = files("offbox").joinpath("_data", "providers", name)
+    if not resource.is_dir():
+        raise PreconditionError(f"Unknown provider: {name}", f"Install a provider in {custom}.")
+    return load_provider(Path(str(resource)), name)
+
+
+TERRAFORM_LOCK_FILE = ".terraform.lock.hcl"
+
+
+def is_provider_file(relative: Path) -> bool:
+    """Whether a path inside a provider folder belongs in a host workspace; local tool state and tests do not."""
+    for part in relative.parts:
+        if part.startswith(".terraform") and part != TERRAFORM_LOCK_FILE:
+            return False
+        if ".tfstate" in part or part.endswith((".tfplan", ".swp", "~")) or part in {".gitkeep", ".DS_Store", "offbox.auto.tfvars.json"}:
+            return False
+    return relative.parts[0] != "tests"
+
+
+def copy_provider(provider: Provider, destination: Path) -> str:
+    sources = sorted(path for path in provider.directory.rglob("*") if is_provider_file(path.relative_to(provider.directory)))
+    if any(path.is_symlink() for path in sources):
+        raise PreconditionError("Provider contains symbolic links", "Replace provider symlinks with regular files before creating a host.")
+    try:
+        destination.mkdir(mode=0o700, parents=True)
+        for source in sources:
+            target = destination / source.relative_to(provider.directory)
+            if source.is_dir():
+                target.mkdir(mode=0o700, exist_ok=True)
+            elif source.is_file():
+                target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+                shutil.copy2(source, target)
+        digest = hashlib.sha256()
+        for path in sorted(destination.rglob("*")):
+            if path.is_file():
+                digest.update(path.relative_to(destination).as_posix().encode() + b"\0" + path.read_bytes() + b"\0")
+        return digest.hexdigest()
+    except OSError as exc:
+        raise PreconditionError(f"Cannot copy provider to {destination}", "Check free space and state directory permissions.") from exc
+
+
+def credentials_env(manifest: ProviderManifest, paths: Paths, env: Mapping[str, str] | None = None) -> dict[str, str]:
+    values = os.environ if env is None else env
+    result: dict[str, str] = {}
+    for name in manifest.credentials_env:
+        if name == "HCLOUD_TOKEN":
+            result[name] = hetzner_token(paths.credentials, values).get_secret_value()
+        elif values.get(name):
+            result[name] = values[name]
+        else:
+            raise PreconditionError(f"Missing provider credential: {name}", f"Set {name} before retrying.")
+    register_secrets(result)
+    return result

offbox/runner.py

@@ -0,0 +1,291 @@
+import logging
+import os
+import selectors
+import shlex
+import signal
+import subprocess
+import time
+from collections.abc import Mapping, Sequence
+from contextlib import suppress
+from pathlib import Path
+from typing import IO, Protocol
+
+from pydantic import BaseModel, ConfigDict
+
+from offbox import console
+from offbox.errors import CommandFailed, PreconditionError, ProviderInterrupted
+from offbox.logging import redact, register_secrets
+
+logger = logging.getLogger(__name__)
+
+
+class Result(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    argv: list[str]
+    returncode: int
+    stdout: str = ""
+    stderr: str = ""
+
+
+class StreamProcess(Protocol):
+    def poll(self) -> int | None: ...
+    def wait(self, timeout: float | None = None) -> int | None:
+        """Return None if a bounded wait expires while the process is still running."""
+        ...
+
+    def kill(self) -> None: ...
+
+
+class _SubprocessStream:
+    def __init__(self, process: subprocess.Popen[bytes], *, grouped: bool, started: float, keepalive: IO[bytes] | None = None) -> None:
+        self.process = process
+        self.grouped = grouped
+        self.started = started
+        self.keepalive = keepalive
+        self.finished = False
+
+    def _finish(self, code: int | None) -> int | None:
+        if code is not None and not self.finished:
+            logger.debug("stream exit=%s duration=%.3f", code, time.monotonic() - self.started)
+            self.finished = True
+            if self.keepalive is not None:
+                with suppress(OSError):
+                    self.keepalive.close()
+        return code
+
+    def poll(self) -> int | None:
+        try:
+            return self._finish(self.process.poll())
+        except OSError as exc:
+            raise PreconditionError("Cannot inspect streaming command", "Check the local process and retry.") from exc
+
+    def wait(self, timeout: float | None = None) -> int | None:
+        try:
+            return self._finish(self.process.wait(timeout=timeout))
+        except subprocess.TimeoutExpired:
+            return None
+        except OSError as exc:
+            raise PreconditionError("Cannot wait for streaming command", "Check the local process and retry.") from exc
+
+    def kill(self) -> None:
+        if self.poll() is not None:
+            return
+        try:
+            with suppress(ProcessLookupError):
+                if self.grouped:
+                    os.killpg(self.process.pid, signal.SIGKILL)
+                else:
+                    self.process.kill()
+        except OSError as exc:
+            raise PreconditionError("Cannot stop streaming command", "Stop the local SSH process and inspect the remote command.") from exc
+        if self.wait(timeout=1.0) is None:
+            raise PreconditionError("Cannot reap streaming command within 1s", "Inspect and stop the local SSH process manually.")
+
+
+class Runner(Protocol):
+    def run(
+        self,
+        argv: Sequence[str],
+        *,
+        check: bool = True,
+        capture: bool = True,
+        input: str | None = None,
+        env: Mapping[str, str] | None = None,
+        cwd: Path | None = None,
+        timeout: float | None = None,
+        interruptible: bool = False,
+    ) -> Result: ...
+
+    def stream(self, argv: Sequence[str], *, env: Mapping[str, str] | None = None, stdin: bool = False, tty: bool = False) -> int: ...
+
+    def start_stream(
+        self,
+        argv: Sequence[str],
+        *,
+        env: Mapping[str, str] | None = None,
+        stdin: bool = False,
+        tty: bool = False,
+        keepalive: bool = False,
+    ) -> StreamProcess:
+        """keepalive gives the child a stdin pipe that stays open until this process ends, so the far side sees a client exit."""
+        ...
+
+
+class SubprocessRunner:
+    def __init__(self, verbose: bool = False) -> None:
+        self.verbose = verbose
+
+    def _start(self, argv: Sequence[str], env: Mapping[str, str] | None) -> tuple[float, dict[str, str]]:
+        values = dict(os.environ)
+        if env is not None:
+            values.update(env)
+        if not argv:
+            raise PreconditionError("Empty command", "Supply a command to run.")
+        register_secrets(values)
+        display = shlex.join([redact(arg, values) for arg in argv])
+        logger.debug("command %s", display)
+        if self.verbose:
+            console.command(display)
+        return time.monotonic(), values
+
+    def _log_output(self, name: str, content: bytes, env: Mapping[str, str]) -> None:
+        text = redact(content.decode(errors="replace").rstrip("\r\n"), env)
+        logger.info("%s: %s", name, text)
+        if self.verbose:
+            console.info(text)
+
+    def _run_logged(
+        self, argv: Sequence[str], env: Mapping[str, str], cwd: Path | None, timeout: float | None
+    ) -> subprocess.CompletedProcess[str]:
+        deadline = None if timeout is None else time.monotonic() + timeout
+
+        def remaining() -> float | None:
+            if deadline is None:
+                return None
+            duration = deadline - time.monotonic()
+            if duration <= 0:
+                raise subprocess.TimeoutExpired(list(argv), timeout or 0)
+            return duration
+
+        with subprocess.Popen(
+            list(argv), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=cwd, start_new_session=True
+        ) as process:
+            if process.stdout is None or process.stderr is None:
+                raise PreconditionError("Cannot capture provider output", "Check subprocess pipe support and retry.")
+            stdout, stderr = process.stdout.fileno(), process.stderr.fileno()
+            outputs = {stdout: bytearray(), stderr: bytearray()}
+            pending = {stdout: bytearray(), stderr: bytearray()}
+            names = {stdout: "stdout", stderr: "stderr"}
+            interrupts = 0
+
+            def interrupt() -> None:
+                nonlocal interrupts
+                interrupts += 1
+                if interrupts == 1:
+                    with suppress(ProcessLookupError):
+                        os.killpg(process.pid, signal.SIGINT)
+                    console.warn("Interrupt received; waiting for the provider to save state. Do not close this terminal.")
+                elif interrupts == 2:
+                    console.warn(
+                        "The provider is still finishing and recording created resources. "
+                        "Press Ctrl-C again to force-stop it; untracked cloud resources may keep billing."
+                    )
+                else:
+                    with suppress(ProcessLookupError):
+                        os.killpg(process.pid, signal.SIGKILL)
+                    process.wait()
+                    raise ProviderInterrupted(
+                        "Provider operation was force-stopped before saving its state",
+                        "Check the provider console for resources labelled offbox-host and delete them, then run offbox host destroy NAME.",
+                        killed=True,
+                    )
+
+            try:
+                with selectors.DefaultSelector() as selector:
+                    selector.register(process.stdout, selectors.EVENT_READ)
+                    selector.register(process.stderr, selectors.EVENT_READ)
+                    while selector.get_map():
+                        try:
+                            for key, _ in selector.select(remaining()):
+                                chunk = os.read(key.fd, 65536)
+                                if not chunk:
+                                    selector.unregister(key.fd)
+                                    if pending[key.fd]:
+                                        self._log_output(names[key.fd], bytes(pending[key.fd]), env)
+                                    continue
+                                outputs[key.fd].extend(chunk)
+                                pending[key.fd].extend(chunk)
+                                while b"\n" in pending[key.fd]:
+                                    end = pending[key.fd].index(b"\n") + 1
+                                    self._log_output(names[key.fd], bytes(pending[key.fd][:end]), env)
+                                    del pending[key.fd][:end]
+                        except KeyboardInterrupt:
+                            interrupt()
+                while True:
+                    try:
+                        returncode = process.wait(timeout=remaining())
+                        break
+                    except KeyboardInterrupt:
+                        interrupt()
+                if interrupts:
+                    raise ProviderInterrupted(
+                        "Provider operation interrupted", "Run offbox host list and destroy the interrupted host.", killed=False
+                    )
+            except (OSError, subprocess.TimeoutExpired, KeyboardInterrupt):
+                with suppress(ProcessLookupError):
+                    os.killpg(process.pid, signal.SIGKILL)
+                process.wait()
+                raise
+            return subprocess.CompletedProcess(
+                list(argv), returncode, bytes(outputs[stdout]).decode(errors="replace"), bytes(outputs[stderr]).decode(errors="replace")
+            )
+
+    def run(
+        self,
+        argv: Sequence[str],
+        *,
+        check: bool = True,
+        capture: bool = True,
+        input: str | None = None,
+        env: Mapping[str, str] | None = None,
+        cwd: Path | None = None,
+        timeout: float | None = None,
+        interruptible: bool = False,
+    ) -> Result:
+        start, values = self._start(argv, env)
+        try:
+            if interruptible:
+                completed = self._run_logged(argv, values, cwd, timeout)
+            else:
+                completed = subprocess.run(
+                    list(argv),
+                    capture_output=capture,
+                    input=input,
+                    stdin=subprocess.DEVNULL if input is None else None,
+                    env=values,
+                    cwd=cwd,
+                    timeout=timeout,
+                    text=True,
+                    check=False,
+                )
+        except subprocess.TimeoutExpired as exc:
+            logger.error("command timeout duration=%.3f", time.monotonic() - start)
+            raise PreconditionError(
+                f"Command timed out: {shlex.join([redact(arg, values) for arg in argv])}", "Check connectivity and try again."
+            ) from exc
+        except OSError as exc:
+            logger.error("command launch failed duration=%.3f", time.monotonic() - start)
+            raise PreconditionError(f"Cannot run {argv[0]}", f"Install {argv[0]} and check its permissions.") from exc
+        result = Result(argv=list(argv), returncode=completed.returncode, stdout=completed.stdout or "", stderr=completed.stderr or "")
+        logger.debug("command exit=%s duration=%.3f", result.returncode, time.monotonic() - start)
+        if check and result.returncode:
+            raise CommandFailed([redact(arg, values) for arg in argv], result.returncode, redact(result.stderr, values))
+        return result
+
+    def stream(self, argv: Sequence[str], *, env: Mapping[str, str] | None = None, stdin: bool = False, tty: bool = False) -> int:
+        start, values = self._start(argv, env)
+        try:
+            result = subprocess.run(list(argv), env=values, stdin=None if stdin or tty else subprocess.DEVNULL, check=False)
+        except OSError as exc:
+            logger.error("stream launch failed duration=%.3f", time.monotonic() - start)
+            raise PreconditionError(f"Cannot run {argv[0]}", f"Install {argv[0]} and check its permissions.") from exc
+        logger.debug("stream exit=%s duration=%.3f", result.returncode, time.monotonic() - start)
+        return result.returncode
+
+    def start_stream(
+        self,
+        argv: Sequence[str],
+        *,
+        env: Mapping[str, str] | None = None,
+        stdin: bool = False,
+        tty: bool = False,
+        keepalive: bool = False,
+    ) -> StreamProcess:
+        started, values = self._start(argv, env)
+        source = subprocess.PIPE if keepalive and not (stdin or tty) else None if stdin or tty else subprocess.DEVNULL
+        try:
+            process = subprocess.Popen(list(argv), env=values, stdin=source, start_new_session=not tty)
+        except OSError as exc:
+            logger.error("stream launch failed duration=%.3f", time.monotonic() - started)
+            raise PreconditionError(f"Cannot run {argv[0]}", f"Install {argv[0]} and check its permissions.") from exc
+        return _SubprocessStream(process, grouped=not tty, started=started, keepalive=process.stdin if source == subprocess.PIPE else None)