hox-cli 0.1.0__py3-none-any.whl

hox_cli/config.py

@@ -0,0 +1,205 @@
+"""hox.toml, parsed once into frozen dataclasses."""
+
+import dataclasses
+import re
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+
+FILENAME = "hox.toml"
+
+
+class ConfigError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class Master:
+    url: str
+    cert: Path
+    key: Path
+    # api.vifib.com serves slap.vifib.com's certificate: verify it under that name.
+    api_tls_name: str | None = None
+
+
+@dataclass(frozen=True)
+class Project:
+    reference: str
+
+
+@dataclass(frozen=True)
+class Node:
+    title: str  # on the master; hox looks its reference up there
+    ssh: tuple[str, ...]
+    # Where `hox re6st-token` requests the node's re6st token (panel: "Re6st Registries").
+    re6st_project: str
+    re6st_release: str
+
+
+@dataclass(frozen=True)
+class Check:
+    name: str
+    url: str
+    headers: dict[str, str] = field(default_factory=dict)
+    expect_status: int | None = None
+    expect_header: dict[str, str] = field(default_factory=dict)
+
+
+RAW = "https://raw.githubusercontent.com"
+
+
+def short_ref(ref: str) -> str:
+    """A commit's first nine characters; a branch stays whole."""
+    return ref[:9] if re.fullmatch(r"[0-9a-f]{40}", ref) else ref
+
+
+@dataclass(frozen=True)
+class Software:
+    name: str
+    repo: str
+    checkout: Path
+    branch: str
+    path: str
+    software_type: str
+    product: str
+    instance: str
+    checks: tuple[Check, ...] = ()
+    # The instance's parameters, sent with every request; None keeps those it has.
+    parameters: dict[str, str] | None = None
+    # Services of etc/service that exit once their work is done: their exit isn't a failure.
+    one_shot: tuple[str, ...] = ()
+
+    def raw_url(self, file: str, ref: str | None = None) -> str:
+        return f"{RAW}/{self.repo}/{ref or self.branch}/{file}"
+
+    def pinned_url(self, commit: str) -> str:
+        return self.raw_url(self.path, commit)
+
+    @property
+    def branch_url(self) -> str:
+        return self.raw_url(self.path)
+
+    def ref_of(self, url: str) -> str | None:
+        """The branch or commit of one of this software's release URLs; None for another URL."""
+        prefix, suffix = f"{RAW}/{self.repo}/", f"/{self.path}"
+        if url.startswith(prefix) and url.endswith(suffix):
+            return url[len(prefix) : -len(suffix)]
+        return None
+
+
+@dataclass(frozen=True)
+class Config:
+    root: Path
+    master: Master
+    project: Project
+    node: Node
+    software: dict[str, Software]
+
+    def software_named(self, name: str | None) -> Software:
+        if name is None:
+            if len(self.software) != 1:
+                raise ConfigError(
+                    f"several software in {FILENAME}, name one of: {', '.join(self.software)}"
+                )
+            return next(iter(self.software.values()))
+        if name in self.software:
+            return self.software[name]
+        for sw in self.software.values():
+            if sw.instance == name:
+                return sw
+        raise ConfigError(
+            f"no software or instance {name!r} in {FILENAME} (software: {', '.join(self.software)})"
+        )
+
+    def url_name(self, url: str) -> str:
+        """A software release URL as hox.toml knows it ("sbe branch", "sbe @ 971d944c2"), else the URL."""
+        for sw in self.software.values():
+            if ref := sw.ref_of(url):
+                return (
+                    f"{sw.name} branch"
+                    if ref == sw.branch
+                    else f"{sw.name} @ {short_ref(ref)}"
+                )
+        return url
+
+
+def find(start: Path) -> Path:
+    for directory in (start, *start.parents):
+        if (directory / FILENAME).is_file():
+            return directory / FILENAME
+    raise ConfigError(f"no {FILENAME} in {start} or its parents")
+
+
+def load(path: Path | None = None) -> Config:
+    path = (path or find(Path.cwd())).resolve()
+    try:
+        data = tomllib.loads(path.read_text())
+    except (OSError, tomllib.TOMLDecodeError) as e:
+        raise ConfigError(f"{path}: {e}") from e
+    return parse(data, path.parent)
+
+
+def parse(data: dict, root: Path) -> Config:
+    data = dict(data)
+
+    def resolve(value: str) -> Path:
+        return (root / Path(value).expanduser()).resolve()
+
+    def ssh_args(args: tuple[str, ...]) -> tuple[str, ...]:
+        # The files of ssh's -F (config) and -i (key) resolve against hox.toml's directory.
+        return tuple(
+            str(resolve(a)) if prev in ("-F", "-i") else a
+            for prev, a in zip(("",) + args, args)
+        )
+
+    master = _build(Master, _table(data, "master"), "master")
+    master = dataclasses.replace(
+        master, cert=resolve(master.cert), key=resolve(master.key)
+    )
+    project = _build(Project, _table(data, "project"), "project")
+    node = _build(Node, _table(data, "node"), "node")
+    node = dataclasses.replace(node, ssh=ssh_args(node.ssh))
+
+    software = {}
+    for name, table in _table(data, "software").items():
+        ctx = f"software.{name}"
+        table = dict(table)
+        checks = tuple(
+            _build(Check, c, f"{ctx}.checks[{i}]")
+            for i, c in enumerate(table.pop("checks", []))
+        )
+        if not all(isinstance(v, str) for v in table.get("parameters", {}).values()):
+            raise ConfigError(f"[{ctx}.parameters] values must be strings")
+        sw = _build(Software, {"name": name, **table, "checks": checks}, ctx)
+        software[name] = dataclasses.replace(sw, checkout=resolve(sw.checkout))
+    if not software:
+        raise ConfigError("[software.<name>]: at least one is required")
+    if data:
+        raise ConfigError(f"unknown sections: {', '.join(data)}")
+    return Config(root, master, project, node, software)
+
+
+def _table(data: dict, name: str) -> dict:
+    if name not in data:
+        raise ConfigError(f"[{name}] is required")
+    value = data.pop(name)
+    if not isinstance(value, dict):
+        raise ConfigError(f"[{name}] must be a table")
+    return value
+
+
+def _build(cls, table: dict, ctx: str):
+    fields = {f.name: f for f in dataclasses.fields(cls)}
+    unknown = set(table) - set(fields)
+    if unknown:
+        raise ConfigError(f"[{ctx}] unknown keys: {', '.join(sorted(unknown))}")
+    required = {
+        n
+        for n, f in fields.items()
+        if f.default is dataclasses.MISSING and f.default_factory is dataclasses.MISSING
+    }
+    missing = required - set(table)
+    if missing:
+        raise ConfigError(f"[{ctx}] missing keys: {', '.join(sorted(missing))}")
+    values = {k: tuple(v) if isinstance(v, list) else v for k, v in table.items()}
+    return cls(**values)

hox_cli/deploy.py

@@ -0,0 +1,95 @@
+"""`hox deploy`: wait for a requested instance to be allocated, built, and to pass its checks.
+
+The node's own cron does the work (it builds what is supplied and deploys what
+is allocated): this only waits, says what it waits for, and forces one
+instance run once the build is there.
+"""
+
+import time
+
+from . import dev, status
+from .config import Software
+from .master import Api
+from .node import SLAPGRID_PROMISE_FAIL, Node
+
+ALLOCATION_POLL_SECONDS, ALLOCATION_TIMEOUT_SECONDS = 30, 30 * 60
+BUILD_POLL_SECONDS, BUILD_TIMEOUT_SECONDS = 60, 8 * 3600  # SBE without cache: hours
+CHECKS_POLL_SECONDS, CHECKS_TIMEOUT_SECONDS = 60, 45 * 60  # frontends take a while
+
+
+def wait_allocated(api: Api, sw: Software, out=print, sleep=time.sleep) -> str:
+    def find() -> list[str]:
+        inst = status.instance(api, sw.instance)
+        return [inst.partition] if inst and inst.partition else []
+
+    out(f"{sw.instance}: waiting for allocation")
+    partition = dev.wait_for(
+        f"{sw.instance}'s allocation",
+        find,
+        sleep,
+        ALLOCATION_POLL_SECONDS,
+        ALLOCATION_TIMEOUT_SECONDS,
+    )
+    out(f"{sw.instance}: allocated on {partition}")
+    return partition
+
+
+def wait_built(node: Node, url: str, out=print, sleep=time.sleep) -> None:
+    last = None
+
+    def find() -> list[bool]:
+        nonlocal last
+        [state] = node.facts([url]).software
+        if (state.state, state.last_error) != last:
+            out(
+                f"build of {url}: {state.state}"
+                + (f" (last error: {state.last_error})" if state.last_error else "")
+            )
+            last = (state.state, state.last_error)
+        return [True] if state.state == "completed" else []
+
+    dev.wait_for(
+        f"build of {url}", find, sleep, BUILD_POLL_SECONDS, BUILD_TIMEOUT_SECONDS
+    )
+
+
+def wait_checks(
+    api: Api, sw: Software, node: Node, out=print, sleep=time.sleep
+) -> list[dev.CheckResult]:
+    last = None
+
+    def find() -> list[list[dev.CheckResult]]:
+        nonlocal last
+        results = dev.run_checks(
+            sw.checks, status.require_instance(api, sw).connection, node
+        )
+        failing = [f"{r.name}: {r.detail}" for r in results if not r.ok]
+        if failing and failing != last:
+            out("waiting for: " + "; ".join(failing))
+        last = failing
+        return [] if failing else [results]
+
+    return dev.wait_for(
+        f"{sw.instance}'s checks",
+        find,
+        sleep,
+        CHECKS_POLL_SECONDS,
+        CHECKS_TIMEOUT_SECONDS,
+    )
+
+
+def deploy(
+    api: Api, sw: Software, url: str, node: Node, out=print, sleep=time.sleep
+) -> list[dev.CheckResult]:
+    """After `hox setup` and `hox request`: allocated, built, deployed, checks passing."""
+    partition = wait_allocated(api, sw, out, sleep)
+    wait_built(node, url, out, sleep)
+    # Only our partition: another one's failure (a release still building) isn't ours.
+    if (
+        node.slapgrid(f"instance --force --only-cp {partition}", out)
+        == SLAPGRID_PROMISE_FAIL
+    ):
+        out(
+            "some promises fail (see above); they may pass once the frontends are there"
+        )
+    return wait_checks(api, sw, node, out, sleep)

hox_cli/dev.py

@@ -0,0 +1,162 @@
+"""The dev loop: pushed? served by GitHub? rebuild in place, then check."""
+
+import shlex
+import time
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+
+import httpx2
+
+from .config import Check, Software
+from .node import Node, git
+
+GITHUB_POLL_SECONDS = 15
+GITHUB_TIMEOUT_SECONDS = (
+    6 * 60
+)  # raw.githubusercontent.com caches branch files up to 5 minutes
+CURL_TIMEOUT_SECONDS = 20
+
+
+class DevError(Exception):
+    pass
+
+
+def wait_for(what: str, find, sleep=time.sleep, poll: int = 5, timeout: int = 180):
+    """The first of find()'s results, once there is one."""
+    waited = 0
+    while not (rows := find()):
+        if waited >= timeout:
+            raise DevError(
+                f"{what}: still not there after {waited // 60} min {waited % 60} s"
+            )
+        sleep(poll)
+        waited += poll
+    return rows[0]
+
+
+def require_pushed(sw: Software) -> str:
+    """The local branch head, if origin has it too (asks the remote: no fetch, no write)."""
+    head = git(sw.checkout, "rev-parse", sw.branch).strip()
+    line = git(sw.checkout, "ls-remote", "origin", f"refs/heads/{sw.branch}").strip()
+    remote = line.split()[0] if line else None
+    if remote != head:
+        raise DevError(
+            f"{sw.branch} is at {head[:9]} locally and {remote[:9] if remote else 'absent'} on origin: push it first"
+        )
+    return head
+
+
+def pushed_commit(sw: Software, commit: str) -> str:
+    """The full SHA of commit, if a branch of origin contains it (as last fetched).
+
+    Pinned URLs use it: GitHub also serves short SHAs, but each spelling is
+    another URL for the master and another software directory on the node.
+    """
+    sha = git(sw.checkout, "rev-parse", "--verify", f"{commit}^{{commit}}").strip()
+    if not git(sw.checkout, "branch", "-r", "--contains", sha).strip():
+        raise DevError(
+            f"{commit} is on no branch of origin (as last fetched): push it first"
+        )
+    return sha
+
+
+def software_files(sw: Software, commit: str) -> dict[str, str]:
+    """The files of the software's directory at commit, as the node will fetch them."""
+    directory = str(PurePosixPath(sw.path).parent)
+    names = git(sw.checkout, "ls-tree", "-r", "--name-only", commit, directory).split()
+    return {name: git(sw.checkout, "show", f"{commit}:{name}") for name in names}
+
+
+def stale_files(sw: Software, files: dict[str, str], http: httpx2.Client) -> list[str]:
+    stale = []
+    for name, content in files.items():
+        response = http.get(sw.raw_url(name))
+        if response.status_code != 200 or response.text != content:
+            stale.append(name)
+    return stale
+
+
+def wait_for_github(
+    sw: Software, commit: str, http: httpx2.Client, out=print, sleep=time.sleep
+) -> None:
+    files = software_files(sw, commit)
+    waited = 0
+    while stale := stale_files(sw, files, http):
+        if waited >= GITHUB_TIMEOUT_SECONDS:
+            raise DevError(
+                f"raw.githubusercontent.com still serves old versions of: {', '.join(stale)}"
+            )
+        out(
+            f"raw.githubusercontent.com serves old {', '.join(stale)}, retrying in {GITHUB_POLL_SECONDS}s"
+        )
+        sleep(GITHUB_POLL_SECONDS)
+        waited += GITHUB_POLL_SECONDS
+    out(f"raw.githubusercontent.com serves {commit[:9]} ({len(files)} files)")
+
+
+@dataclass(frozen=True)
+class CheckResult:
+    name: str
+    url: str | None  # None: a connection parameter it needs is not published
+    ok: bool
+    detail: str
+
+
+def run_checks(
+    checks: tuple[Check, ...], connection: dict[str, str], node: Node
+) -> list[CheckResult]:
+    """Run each check with curl on the node: backends are only reachable over re6st."""
+    published = {k: v for k, v in connection.items() if v not in (None, "None", "")}
+    results = []
+    for check in checks:
+        try:
+            url = check.url.format_map(published)
+        except KeyError as e:
+            results.append(
+                CheckResult(
+                    check.name, None, False, f"{e.args[0]} is not published yet"
+                )
+            )
+            continue
+        results.append(evaluate(check, url, node.run(curl_command(check, url))))
+    return results
+
+
+def curl_command(check: Check, url: str) -> str:
+    args = [
+        "curl",
+        "-gsk",
+        "-o",
+        "/dev/null",
+        "-D",
+        "-",
+        "--max-time",
+        str(CURL_TIMEOUT_SECONDS),
+    ]
+    for key, value in check.headers.items():
+        args += ["-H", f"{key}: {value}"]
+    # Always exit 0 so the output is ours to read; curl's own code comes last.
+    return shlex.join([*args, url]) + '; echo "curl-exit=$?"'
+
+
+def evaluate(check: Check, url: str, output: str) -> CheckResult:
+    lines = output.strip().splitlines()
+    curl_exit = int(lines[-1].removeprefix("curl-exit="))
+    if curl_exit != 0:
+        return CheckResult(check.name, url, False, f"curl exit {curl_exit}")
+    status_line = next(line for line in lines if line.startswith("HTTP/"))
+    status = int(status_line.split()[1])
+    headers = {}
+    for line in lines:
+        if ":" in line and not line.startswith("HTTP/"):
+            key, value = line.split(":", 1)
+            headers[key.strip().lower()] = value.strip()
+    problems = []
+    if check.expect_status is not None and status != check.expect_status:
+        problems.append(f"status {status}, expected {check.expect_status}")
+    for key, value in check.expect_header.items():
+        if headers.get(key.lower()) != value:
+            problems.append(f"{key}: {headers.get(key.lower())!r}, expected {value!r}")
+    return CheckResult(
+        check.name, url, not problems, "; ".join(problems) or f"status {status}"
+    )

hox_cli/lock.py

@@ -0,0 +1,198 @@
+"""From a project's uv.lock to buildout's [versions]: what the project needs at run
+time on a SlapOS node, pinned as uv resolved it, and what buildout can't install
+as is."""
+
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+from packaging import tags
+from packaging.markers import Marker
+from packaging.utils import canonicalize_name, parse_wheel_filename
+
+
+# buildout's own: their versions are SlapOS's (stack/slapos.cfg), whatever the app wants.
+BUILDOUT_TOOLS = frozenset({"pip", "setuptools", "wheel", "zc-buildout"})
+
+# Built by SlapOS's stack itself, from source, for slapos.cookbook (stack/slapos.cfg):
+# a :whl pin breaks that build.
+STACK_BUILT = frozenset({"cffi", "lxml", "pyyaml"})
+
+
+class LockError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class Pin:
+    name: str
+    version: str
+    sdist: bool
+    wheel: (
+        str | None
+    )  # "pure", "binary" (manylinux, for the node), or none for the node
+
+    @property
+    def stack_built(self) -> bool:
+        return canonicalize_name(self.name) in STACK_BUILT
+
+    @property
+    def constraint(self) -> str:
+        # A wheel installs without a build: buildout builds sdists with the build's
+        # setuptools (74 on SlapOS 1.0), which many recent sdists are too new for.
+        return (
+            f"{self.version}:whl"
+            if self.wheel and not self.stack_built
+            else self.version
+        )
+
+
+def node_tags(python: str) -> frozenset[tags.Tag]:
+    """The wheel tags a SlapOS node's CPython `python` installs: manylinux x86_64
+    (the node's glibc, Debian 12, is 2.36)."""
+    version = tuple(int(n) for n in python.split(".")[:2])
+    platforms = [f"manylinux_2_{minor}_x86_64" for minor in range(36, 4, -1)]
+    platforms += [
+        "manylinux2014_x86_64",
+        "manylinux2010_x86_64",
+        "manylinux1_x86_64",
+        "linux_x86_64",
+    ]
+    return frozenset([
+        *tags.cpython_tags(version, platforms=platforms),
+        *tags.compatible_tags(version, platforms=platforms),
+    ])
+
+
+def wheel_kind(filenames: list[str], supported: frozenset[tags.Tag]) -> str | None:
+    found = {t for f in filenames for t in parse_wheel_filename(f)[3]} & supported
+    if any(t.platform == "any" for t in found):
+        return "pure"
+    return "binary" if found else None
+
+
+def environment(python: str) -> dict[str, str]:
+    """Marker values on a SlapOS node: Linux x86_64, CPython `python` (3.12)."""
+    return {
+        "python_version": ".".join(python.split(".")[:2]),
+        "python_full_version": python,
+        "implementation_name": "cpython",
+        "platform_python_implementation": "CPython",
+        "sys_platform": "linux",
+        "platform_system": "Linux",
+        "platform_machine": "x86_64",
+        "os_name": "posix",
+    }
+
+
+def runtime_pins(lock: dict, python: str) -> list[Pin]:
+    """The root project's run-time dependencies (no dependency groups), with the
+    extras they ask for, whose markers hold on the node."""
+    env = environment(python)
+    supported = node_tags(python)
+    packages: dict[str, list[dict]] = {}
+    for p in lock["package"]:
+        packages.setdefault(p["name"], []).append(p)
+    roots = [p for p in lock["package"] if {"editable", "virtual"} & p["source"].keys()]
+    if len(roots) != 1:
+        raise LockError(
+            f"expected one editable or virtual root package, found {len(roots)}"
+        )
+    root = roots[0]
+
+    def holds(dep: dict, extra: str = "") -> bool:
+        return "marker" not in dep or Marker(dep["marker"]).evaluate(
+            env | {"extra": extra}
+        )
+
+    def package(dep: dict) -> dict:
+        found = packages[dep["name"]]
+        if len(found) > 1:  # uv forked the resolution: the dependency says which
+            found = [p for p in found if p["version"] == dep["version"]]
+        return found[0]
+
+    seen: set[tuple[str, str]] = set()
+    pins: dict[str, Pin] = {}
+    todo = [(root, "")]
+    while todo:
+        p, extra = todo.pop()
+        if (p["name"], extra) in seen:
+            continue
+        seen.add((p["name"], extra))
+        deps = (
+            p.get("optional-dependencies", {}).get(extra, [])
+            if extra
+            else p.get("dependencies", [])
+        )
+        for dep in deps:
+            if not holds(dep, extra):
+                continue
+            q = package(dep)
+            todo += [(q, "")] + [(q, e) for e in dep.get("extra", [])]
+        if p is not root and not extra and p["name"] not in BUILDOUT_TOOLS:
+            wheels = [w["url"].rsplit("/", 1)[1] for w in p.get("wheels", [])]
+            pins[p["name"]] = Pin(
+                p["name"], p["version"], "sdist" in p, wheel_kind(wheels, supported)
+            )
+    return sorted(pins.values(), key=lambda pin: pin.name)
+
+
+def versions_cfg(path: Path, python: str) -> str:
+    """A [versions] section for buildout, with what needs attention as comments."""
+    return render_versions(
+        runtime_pins(tomllib.loads(path.read_text()), python),
+        f"{path.name} by `hox versions`",
+        python,
+    )
+
+
+def render_versions(pins: list[Pin], source: str, python: str) -> str:
+    binary = [p.name for p in pins if p.wheel == "binary" and not p.stack_built]
+    stack = [p.name for p in pins if p.stack_built]
+    compiled = [p.name for p in pins if not p.wheel and p.sdist]
+    unbuildable = [p.name for p in pins if not p.wheel and not p.sdist]
+    lines = [
+        f"# From {source}, for CPython {python} on Linux x86_64.",
+        "# Packages with a wheel for the node are installed from it (:whl).",
+    ]
+    if binary:
+        lines += [
+            "# From binary (manylinux) wheels, with their C libraries bundled, not",
+            "# SlapOS's: a security fix in one comes with a new release of the package.",
+            *_wrapped(binary),
+        ]
+    if stack:
+        lines += [
+            "# Built by SlapOS's stack (slapos.cookbook), from source, with its recipe:",
+            *_wrapped(stack),
+        ]
+    if compiled:
+        lines += [
+            "# Built from their sdist (no wheel for the node): those with C or Rust",
+            "# code need their libraries and build tools in the build:",
+            *_wrapped(compiled),
+        ]
+    if unbuildable:
+        lines += [
+            "# No sdist and no wheel for the node, so buildout can't install them:",
+            "# replace them (another extra, or a SlapOS component):",
+            *_wrapped(unbuildable),
+        ]
+    width = max(len(p.name) for p in pins)
+    return "\n".join([
+        *lines,
+        "",
+        "[versions]",
+        *(f"{p.name:<{width}} = {p.constraint}" for p in pins),
+        "",
+    ])
+
+
+def _wrapped(names: list[str]) -> list[str]:
+    lines, line = [], "#  "
+    for name in names:
+        if len(line) + len(name) > 76:
+            lines.append(line)
+            line = "#  "
+        line += f" {name}"
+    return [*lines, line]