hox-cli 0.1.0__py3-none-any.whl

hox_cli/master.py

@@ -0,0 +1,371 @@
+"""The SlapOS master: its API (hateoas) for reads and forms, and the slap protocol."""
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from urllib.parse import quote, urljoin, urlsplit
+from xml.etree import ElementTree
+from xml.sax.saxutils import escape
+
+import httpx2
+
+from .config import Master
+
+TIMEOUT = 60
+
+
+class MasterError(Exception):
+    pass
+
+
+def expand(template: str, params: Mapping[str, str | int | list]) -> str:
+    """RFC 6570 form-style query expansion ({&a,b*}), the only kind the API uses."""
+
+    def replace(match: re.Match) -> str:
+        operator, names = match.group(1)[:1], match.group(1)[1:].split(",")
+        if operator not in ("&", "?"):
+            raise MasterError(f"unsupported URI template expression: {match.group(0)}")
+        pairs = []
+        for name in names:
+            explode = name.endswith("*")
+            name = name.rstrip("*")
+            value = params.get(name)
+            if value is None:
+                continue
+            values = value if isinstance(value, list) else [value]
+            if not explode and isinstance(value, list):
+                values = [",".join(map(str, value))]
+            pairs += [f"{name}={quote(str(v), safe='')}" for v in values]
+        if not pairs:
+            return ""
+        return operator + "&".join(pairs)
+
+    return re.sub(r"\{([^}]+)\}", replace, template)
+
+
+class Api:
+    def __init__(self, client: httpx2.Client, root: dict, tls_name: str | None = None):
+        self._client = client
+        self._extensions = {"sni_hostname": tls_name} if tls_name else {}
+        self._root = root
+
+    @classmethod
+    def connect(
+        cls, master: Master, transport: httpx2.BaseTransport | None = None
+    ) -> "Api":
+        client = httpx2.Client(
+            cert=(str(master.cert), str(master.key)),
+            timeout=TIMEOUT,
+            transport=transport,
+        )
+        url = _get(client, urljoin(master.url, "getHateoasUrl")).text.strip()
+        if urlsplit(url).scheme != "https":
+            raise MasterError(f"getHateoasUrl returned {url!r}")
+        extensions = (
+            {"sni_hostname": master.api_tls_name} if master.api_tls_name else {}
+        )
+        root = _get(client, url, extensions).json()
+        return cls(client, root, master.api_tls_name)
+
+    def _get_json(self, url: str) -> dict:
+        return _get(self._client, url, self._extensions).json()
+
+    def _template(self, name: str) -> str:
+        try:
+            return self._root["_links"][name]["href"]
+        except KeyError:
+            raise MasterError(f"API root has no {name!r} link") from None
+
+    def search(self, query: str, select: list[str], limit: int = 100) -> list[dict]:
+        url = expand(
+            self._template("raw_search"),
+            {"query": query, "select_list": select, "limit": limit},
+        )
+        rows = self._get_json(url)["_embedded"]["contents"]
+        for row in rows:
+            row.pop("_links", None)
+        return rows
+
+    def get(self, relative_url: str, view: str | None = None) -> dict:
+        params = {"relative_url": relative_url}
+        if view:
+            params["view"] = view
+        return self._get_json(expand(self._template("traverse"), params))
+
+    def act(
+        self, relative_url: str, action: str, fields: dict[str, str] | None = None
+    ) -> dict | None:
+        """Submit one of a document's actions as the panel does: a fast input dialog
+        (add_slapos_software_type), a workflow transition (validate_action) or a script.
+
+        `fields` are keyed as the form keys them (field_your_software_type); every
+        other field of the form is sent with its default. Returns the master's
+        answer when it is JSON (a script may answer with a form of its results).
+        """
+        links = [
+            link
+            for name, value in self.get(relative_url, "view")["_links"].items()
+            if name.startswith("action_")
+            for link in (value if isinstance(value, list) else [value])
+        ]
+        href = next(
+            (link["href"] for link in links if link.get("name") == action), None
+        )
+        if href is None:
+            names = sorted({link.get("name") for link in links})
+            raise MasterError(
+                f"{relative_url} has no action {action!r}; it has: {', '.join(names)}"
+            )
+        form = self._get_json(href)["_embedded"]["_view"]
+        actions = form["_actions"]
+        data = {
+            f["key"]: f["default"] or ""
+            for f in form.values()
+            if isinstance(f, dict) and "key" in f
+        }
+        # The method is in _actions.put for dialogs, next to it for scripts.
+        data["dialog_method"] = actions["put"].get("action") or actions["action"]
+        return self._post(actions["put"]["href"], data | (fields or {}))
+
+    def edit(
+        self, relative_url: str, fields: dict[str, str], view: str = "view"
+    ) -> None:
+        """Change fields of one of a document's views, keyed as the form keys them (field_my_title).
+
+        Base_edit skips the fields absent from the POST: the others keep their values.
+        """
+        form = self.get(relative_url, view)["_embedded"]["_view"]
+        self._post(
+            form["_actions"]["put"]["href"],
+            {"form_id": form["form_id"]["default"], **fields},
+        )
+
+    def _post(self, url: str, data: dict) -> dict | None:
+        try:
+            response = self._client.post(
+                url,
+                data=data,
+                extensions=self._extensions,
+                headers={"Accept": "application/hal+json"},
+            )
+        except httpx2.TransportError as e:
+            raise MasterError(f"POST {url}: {e}") from e
+        if response.is_error:
+            raise MasterError(
+                f"POST {url}: HTTP {response.status_code} {response.text[:200]}"
+            )
+        if "json" not in response.headers.get(
+            "content-type", ""
+        ):  # application/json or application/hal+json
+            return None  # a redirect
+        body = response.json()
+        if body.get("portal_status_level") == "error":
+            raise MasterError(f"POST {url}: {body.get('portal_status_message')}")
+        return body
+
+
+def _get(
+    client: httpx2.Client, url: str, extensions: dict | None = None
+) -> httpx2.Response:
+    try:
+        response = client.get(url, extensions=extensions or {})
+    except httpx2.TransportError as e:
+        raise MasterError(f"GET {url}: {e}") from e
+    if response.is_error:
+        raise MasterError(
+            f"{response.request.method} {response.url}: HTTP {response.status_code} {response.text[:200]}"
+        )
+    return response
+
+
+def quoted(value: str) -> str:
+    """A value for an ERP5 catalog query (title:="...")."""
+    return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+# The slap protocol: form POSTs to master.url, some fields marshalled as
+# slapos.core does (slapos.util.dumps, golden values in tests/test_master.py).
+
+
+def marshal(value: dict[str, str] | str | bool) -> str:
+    def text(s: str) -> str:
+        # slapos.core escapes twice (xml_marshaller, then serialization); the master expects it.
+        return f"<string>{escape(escape(s))}</string>"
+
+    if isinstance(value, bool):
+        body = f"<bool>{int(value)}</bool>"
+    elif isinstance(value, str):
+        body = text(value)
+    elif not value:
+        body = '<dictionary id="i2"/>'
+    else:
+        body = (
+            '<dictionary id="i2">'
+            + "".join(text(k) + text(v) for k, v in value.items())
+            + "</dictionary>"
+        )
+    return f"<marshal>{body}</marshal>"
+
+
+_STATES = {
+    "start_requested": "started",
+    "stop_requested": "stopped",
+    "destroy_requested": "destroyed",
+}
+
+
+@dataclass(frozen=True)
+class Request:
+    """An instance request, as sent to requestComputerPartition."""
+
+    software_release: str
+    reference: str
+    software_type: str
+    filter: dict[str, str]
+    parameters: dict[str, str] = field(default_factory=dict)
+    shared: bool = False
+    state: str = "started"
+    project: str | None = (
+        None  # without it, the master guesses (from the filter or the release)
+    )
+
+    def form(self) -> dict[str, str]:
+        project = {"project_reference": self.project} if self.project else {}
+        return project | {
+            "software_release": self.software_release,
+            "partition_reference": self.reference,
+            "software_type": self.software_type,
+            "partition_parameter_xml": marshal(self.parameters),
+            "filter_xml": marshal(self.filter),
+            "state": marshal(self.state),
+            "shared_xml": marshal(self.shared),
+        }
+
+
+@dataclass(frozen=True)
+class ComputeNode:
+    reference: str
+    relative_url: str
+
+
+def nodes_titled(api: Api, project: str, title: str) -> list[ComputeNode]:
+    rows = api.search(
+        f'portal_type:"Compute Node" AND validation_state:validated AND follow_up_reference:={quoted(project)}'
+        f" AND title:={quoted(title)}",
+        ["reference", "relative_url"],
+    )
+    return [ComputeNode(r["reference"], r["relative_url"]) for r in rows]
+
+
+def find_node(api: Api, project: str, title: str) -> ComputeNode:
+    """Our compute node, by its title in the project (its reference changes with each registration)."""
+    nodes = nodes_titled(api, project, title)
+    if len(nodes) != 1:
+        found = f"{len(nodes)} of them" if nodes else "none (`hox node-certificate`)"
+        raise MasterError(f"compute node {title!r} in {project}: {found}")
+    return nodes[0]
+
+
+def stored_request(api: Api, reference: str) -> Request | None:
+    """The request an instance tree was created with, read back from the master.
+
+    The master answers 500 to a request whose URL, type or filter differ from
+    the original: re-sending this one (with another state) is what works.
+    """
+    trees = api.search(
+        f'portal_type:"Instance Tree" AND validation_state:validated AND title:={quoted(reference)}',
+        [
+            "url_string",
+            "source_reference",
+            "sla_xml",
+            "text_content",
+            "root_slave",
+            "slap_state",
+            "follow_up_reference",
+        ],
+    )
+    if not trees:
+        return None
+    tree = trees[0]
+    return Request(
+        software_release=tree["url_string"],
+        reference=reference,
+        software_type=tree["source_reference"],
+        filter=instance_xml(tree["sla_xml"]),
+        parameters=instance_xml(tree["text_content"]),
+        shared=bool(tree["root_slave"]),
+        state=_STATES[tree["slap_state"]],
+        project=tree["follow_up_reference"],
+    )
+
+
+def instance_xml(xml: str | None) -> dict[str, str]:
+    """<instance><parameter id="k">v</parameter>…</instance> -> {k: v}."""
+    if not xml:
+        return {}
+    params = {}
+    for p in ElementTree.fromstring(xml.encode()).iter("parameter"):
+        key = p.get("id")
+        if key is None:
+            raise MasterError(f"instance XML: a parameter has no id: {xml[:200]}")
+        params[key] = p.text or ""
+    return params
+
+
+class Slap:
+    def __init__(self, client: httpx2.Client, url: str):
+        self._client = client
+        self._url = url
+
+    @classmethod
+    def connect(
+        cls, master: Master, transport: httpx2.BaseTransport | None = None
+    ) -> "Slap":
+        client = httpx2.Client(
+            cert=(str(master.cert), str(master.key)),
+            timeout=TIMEOUT,
+            transport=transport,
+        )
+        return cls(client, master.url)
+
+    def _post(self, path: str, data: dict[str, str]) -> httpx2.Response:
+        url = urljoin(self._url, path)
+        try:
+            return self._client.post(url, data=data)
+        except httpx2.TransportError as e:
+            raise MasterError(f"POST {url}: {e}") from e
+
+    def supply(self, url: str, node: str, state: str = "available") -> None:
+        response = self._post(
+            "supplySupply", {"url": url, "computer_id": node, "state": state}
+        )
+        if response.is_error:
+            raise MasterError(
+                f"supply {url} on {node}: HTTP {response.status_code} {response.text[:200]}"
+            )
+
+    def request(self, request: Request) -> bool:
+        """Send an instance request. True: the master processed it; False: still provisioning (HTTP 408)."""
+        response = self._post("requestComputerPartition", request.form())
+        if (
+            response.status_code == 408
+        ):  # what the master answers while it provisions, not a timeout
+            return False
+        if response.status_code == 500:
+            raise MasterError(
+                f"request {request.reference}: HTTP 500. The master answers that to a request whose URL,"
+                " software type or filter differ from the instance's original one."
+            )
+        if response.is_error:
+            raise MasterError(
+                f"request {request.reference}: HTTP {response.status_code} {response.text[:200]}"
+            )
+        return True
+
+
+def next_name(reference: str) -> str:
+    """mynij-proxy-3 -> mynij-proxy-4; mynij-proxy -> mynij-proxy-2."""
+    base, _, number = reference.rpartition("-")
+    return (
+        f"{base}-{int(number) + 1}" if base and number.isdigit() else f"{reference}-2"
+    )

hox_cli/node.py

@@ -0,0 +1,302 @@
+"""The compute node, over SSH, and the local git checkouts."""
+
+import hashlib
+import re
+import shlex
+import subprocess
+import time
+from collections import deque
+from dataclasses import dataclass
+from pathlib import Path
+
+SOFTWARE_ROOT = "/opt/slapgrid"
+SOFTWARE_LOG = "/opt/slapos/log/slapos-node-software.log"
+INSTANCE_LOG = "/opt/slapos/log/slapos-node-instance.log"
+LOG_LINES = 2000
+SEPARATOR = "----hox----"
+
+
+class NodeError(Exception):
+    pass
+
+
+def software_dir(url: str) -> str:
+    return f"{SOFTWARE_ROOT}/{hashlib.md5(url.encode()).hexdigest()}"
+
+
+@dataclass(frozen=True)
+class SoftwareState:
+    url: str
+    state: str  # "completed", "building" (dir present, not completed) or "absent"
+    last_error: (
+        str | None
+    )  # last buildout error for this dir, if the last run of it failed
+
+
+@dataclass(frozen=True)
+class NodeFacts:
+    software: list[SoftwareState]
+    failing_promises: list[str]  # from the last `slapos node instance` run
+
+
+# Exit codes of `slapos node software|instance`.
+SLAPGRID_OK, SLAPGRID_FAIL, SLAPGRID_PROMISE_FAIL = 0, 1, 2
+SLAPGRID_BUSY = 10  # another run (the node's cron) holds the pidfile
+BUSY_RETRY_SECONDS = 15
+BUSY_TIMEOUT_SECONDS = 15 * 60
+SSH_FAILED = 255  # ssh's own failure, not the remote command's
+TAIL_LINES = 10  # of a failed run, in the error
+
+# What `hox rebuild` shows of a slapgrid run: parts that change (buildout says
+# "Updating x." for every unchanged one), errors and the end of each run.
+_SHOWN = re.compile(
+    r"(Installing|Uninstalling) \S+\.$|Error|While:|Failed to run buildout"
+    r"|Finished (software releases|computer partitions)|slappart\d+\["
+)
+
+
+def shown(line: str) -> bool:
+    return bool(_SHOWN.search(line))
+
+
+@dataclass(frozen=True)
+class Partition:
+    name: str
+    memory_mb: int  # the resident size of its user's processes
+    running: int
+    down: list[
+        str
+    ]  # services that should run and don't, as `service_name` shortens them
+
+
+@dataclass(frozen=True)
+class System:
+    uptime_seconds: int
+    partitions: list[Partition]
+    memory_available_mb: int
+    memory_total_mb: int
+    disk_free_mb: int
+    disk_total_mb: int
+
+
+class Node:
+    def __init__(
+        self,
+        ssh: tuple[str, ...],
+        run=subprocess.run,
+        popen=subprocess.Popen,
+        sleep=time.sleep,
+    ):
+        self._ssh = ssh
+        self._run = run
+        self._popen = popen
+        self._sleep = sleep
+
+    def run(self, command: str) -> str:
+        result = self._run([*self._ssh, command], capture_output=True, text=True)
+        if result.returncode == SSH_FAILED:
+            raise NodeError(
+                f"{self._ssh[0]} {self._ssh[-1]}: {result.stderr.strip()[-500:].replace(chr(10), '; ')}"
+            )
+        if result.returncode != 0:
+            raise NodeError(
+                f"{shlex.join(self._ssh)} {command!r}: exit {result.returncode}: {result.stderr.strip()[-500:]}"
+            )
+        return result.stdout
+
+    def stream(self, command: str, keep, out=print) -> tuple[int, list[str]]:
+        """Run command, pass the lines that `keep` accepts to `out`.
+
+        Returns its exit code and its last lines, shown or not (ssh's own errors included).
+        """
+        last: deque[str] = deque(maxlen=TAIL_LINES)
+        with self._popen(
+            [*self._ssh, command],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        ) as proc:
+            assert proc.stdout is not None  # stdout=PIPE
+            for line in proc.stdout:
+                line = line.rstrip("\n")
+                last.append(line)
+                if keep(line):
+                    out(line)
+        return proc.returncode, list(last)
+
+    def interactive(self, command: str) -> int:
+        """Run command with the terminal attached (tail -F), return its exit code."""
+        return self._run([*self._ssh, command]).returncode
+
+    def slapgrid(self, args: str, out=print) -> int:
+        """`slapos node <args>`, waiting while the node's own run holds the lock.
+
+        Returns SLAPGRID_OK or SLAPGRID_PROMISE_FAIL; raises on anything else.
+        """
+        waited = 0
+        while True:
+            code, last = self.stream(f"slapos node {args}", shown, out)
+            if code != SLAPGRID_BUSY:
+                break
+            if waited >= BUSY_TIMEOUT_SECONDS:
+                raise NodeError(
+                    f"slapos node {args}: another slapos run kept the lock for {waited // 60} min"
+                )
+            out(f"another slapos run is going on, retrying in {BUSY_RETRY_SECONDS}s")
+            self._sleep(BUSY_RETRY_SECONDS)
+            waited += BUSY_RETRY_SECONDS
+        if code not in (SLAPGRID_OK, SLAPGRID_PROMISE_FAIL):
+            what = "ssh failed" if code == SSH_FAILED else f"exit {code}"
+            raise NodeError(
+                f"slapos node {args}: {what}; its last lines:\n" + "\n".join(last)
+            )
+        return code
+
+    def facts(self, urls: list[str]) -> NodeFacts:
+        dirs = [software_dir(u) for u in urls]
+        script = "; ".join([
+            f"for d in {' '.join(dirs)}; do"
+            " if [ -f $d/.completed ]; then echo $d completed;"
+            " elif [ -d $d ]; then echo $d building; else echo $d absent; fi; done"
+            if dirs
+            else "true",
+            f"echo {SEPARATOR}",
+            f"tail -n {LOG_LINES} {SOFTWARE_LOG}",
+            f"echo {SEPARATOR}",
+            f"tail -n {LOG_LINES} {INSTANCE_LOG}",
+        ])
+        return parse_facts(urls, self.run(script))
+
+    def system(self) -> System:
+        return parse_system(self.run(_SYSTEM_SCRIPT))
+
+
+_SYSTEM_SCRIPT = f"; echo {SEPARATOR}; ".join([
+    "cut -d' ' -f1 /proc/uptime",
+    "ls -d /srv/slapgrid/slappart*",
+    "slapos node status 2>/dev/null",
+    "ps -eo user:32,rss --no-headers",
+    "free -m",
+    "df -m /srv/slapgrid",
+])
+_SERVICE = re.compile(r"(slappart\d+):(\S+)\s+([A-Z]+)")
+
+
+def service_name(program: str) -> str:
+    """web-2aa681dd2c1eb6c7422d9a9becca20d9-on-watch -> web."""
+    return re.sub(r"(-[0-9a-f]{32})?(-on-watch)?$", "", program)
+
+
+def parse_system(output: str) -> System:
+    uptime, names, services, processes, free, df = output.split(SEPARATOR + "\n")
+    running: dict[str, int] = {}
+    down: dict[str, list[str]] = {}
+    for partition, program, state in _SERVICE.findall(services):
+        # Supervisor restarts nothing here and shows no exit code, so a crash also reads
+        # EXITED: only -on-watch services must stay up, the others run once or wait their turn.
+        if state == "RUNNING":
+            running[partition] = running.get(partition, 0) + 1
+        elif program.endswith("-on-watch") or state not in ("EXITED", "STOPPED"):
+            down.setdefault(partition, []).append(service_name(program))
+    memory_kb: dict[str, int] = {}
+    for user, rss in (line.split() for line in processes.splitlines()):
+        if user.startswith("slapuser"):
+            partition = user.replace("slapuser", "slappart")
+            memory_kb[partition] = memory_kb.get(partition, 0) + int(rss)
+    # free -m: total used free shared buff/cache available; df -m: fs total used available.
+    memory = free.splitlines()[1].split()
+    disk = df.splitlines()[1].split()
+    partitions = sorted(
+        (name.rsplit("/", 1)[1] for name in names.split()),
+        key=lambda name: int(name.removeprefix("slappart")),
+    )
+    return System(
+        uptime_seconds=int(float(uptime)),
+        partitions=[
+            Partition(
+                name,
+                memory_kb.get(name, 0) // 1024,
+                running.get(name, 0),
+                down.get(name, []),
+            )
+            for name in partitions
+        ],
+        memory_available_mb=int(memory[6]),
+        memory_total_mb=int(memory[1]),
+        disk_free_mb=int(disk[3]),
+        disk_total_mb=int(disk[1]),
+    )
+
+
+def parse_facts(urls: list[str], output: str) -> NodeFacts:
+    states, software_log, instance_log = output.split(SEPARATOR + "\n")
+    by_dir = dict(line.split(" ", 1) for line in states.splitlines())
+    errors = last_buildout_errors(software_log)
+    software = []
+    for url in urls:
+        d = software_dir(url)
+        state = by_dir.get(d, "absent")
+        software.append(
+            SoftwareState(url, state, errors.get(d) if state != "completed" else None)
+        )
+    return NodeFacts(software, failing_promises(instance_log))
+
+
+_FAILED = re.compile(r"Failed to run buildout profile in directory '([^']+)'")
+_ERROR = re.compile(r"INFO\s+(Error: .*)")
+_RUN_START = re.compile(r"Processing software releases\.\.\.")
+
+
+def last_buildout_errors(log: str) -> dict[str, str]:
+    """Software dir -> the 'Error:' line of its last failed buildout run.
+
+    A later successful run in the same log clears nothing here: callers only
+    look at dirs that are not completed.
+    """
+    errors: dict[str, str] = {}
+    last_error = None
+    for line in log.splitlines():
+        if _RUN_START.search(line):
+            last_error = None
+        elif m := _ERROR.search(line):
+            last_error = m.group(1)
+        elif m := _FAILED.search(line):
+            errors[m.group(1)] = (
+                last_error or "buildout failed (no 'Error:' line in the log tail)"
+            )
+    return errors
+
+
+def failing_promises(log: str) -> list[str]:
+    """Promise failures of the last finished `slapos node instance` run.
+
+    A run logs "Finished computer partitions.", then, if promises failed,
+    "Error with promises for the following partitions:" and one line each.
+    """
+    lines = log.splitlines()
+    finished = [
+        i for i, line in enumerate(lines) if "Finished computer partitions." in line
+    ]
+    if not finished:
+        return []
+    result = []
+    in_block = False
+    for line in lines[finished[-1] + 1 :]:
+        if re.search(r"INFO\s+=+$", line):
+            continue
+        if "Error with promises for the following partitions:" in line:
+            in_block = True
+        elif in_block and (m := re.search(r"INFO\s{3,}(slappart\S+: .*)", line)):
+            result.append(m.group(1))
+        else:
+            break
+    return result
+
+
+def git(checkout: Path, *args: str) -> str:
+    result = subprocess.run(
+        ["git", "-C", str(checkout), *args], capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        raise NodeError(f"git -C {checkout} {' '.join(args)}: {result.stderr.strip()}")
+    return result.stdout