hox-cli 0.1.0__py3-none-any.whl

hox_cli/report.py

@@ -0,0 +1,208 @@
+"""`hox build-report`: how long a software's builds took on the node, and the disk they use, as Markdown."""
+
+import re
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+from .config import Config, Software
+from .node import SEPARATOR, SOFTWARE_ROOT, Node, software_dir
+
+LOG_DIR = "/opt/slapos/log"
+SHARED = f"{SOFTWARE_ROOT}/shared"
+CRON = "/etc/cron.d/slapos-node"
+LONGEST = 15  # parts listed per run; the others are summed up
+LARGEST = 10  # shared parts listed
+
+# What a `slapos node software` log says about a build (slapgrid and buildout lines).
+_EVENT = re.compile(
+    r"^\[(?P<time>[\d-]+ [\d:]+),\d+\] \w+ +(?:"
+    r"Installing software release (?P<release>\S+) \.\.\."
+    r"|Installing (?P<part>[\w.+-]+)\."
+    r"|.*\"POST /(?P<available>availableSoftwareRelease) .*"
+    r"|(?P<failed>Failed to run buildout profile) .*"
+    r"|(?P<finished>Finished software releases\.)"
+    r"|(?P<processing>Processing software releases\.\.\.))$"
+)
+
+
+def utc(text: str) -> datetime:
+    """A time of the node's logs or `date -u`: the node runs in UTC."""
+    return datetime.strptime(text, "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC)
+
+
+_GREP = (
+    r"Installing software release |INFO +Installing [A-Za-z0-9_.+-]+\.$"
+    r"|POST /availableSoftwareRelease|Failed to run buildout profile|(Finished|Processing) software releases"
+)
+
+
+@dataclass
+class Run:
+    """One `slapos node software` attempt at a release URL."""
+
+    url: str
+    start: datetime
+    end: datetime | None = None  # None: still running when the log was read
+    result: str = (
+        "building"  # "completed", "failed", "stopped" (killed, no result) or "building"
+    )
+    parts: list[tuple[str, datetime]] = field(
+        default_factory=list
+    )  # (part, when its install started)
+
+    def part_times(self, now: datetime) -> list[tuple[str, float]]:
+        """(part, seconds), each part lasting until the next one starts or the run ends."""
+        ends = [t for _, t in self.parts[1:]] + [self.end or now]
+        return [
+            (name, (end - start).total_seconds())
+            for (name, start), end in zip(self.parts, ends)
+        ]
+
+
+def parse_runs(log: str) -> list[Run]:
+    runs: list[Run] = []
+    current: Run | None = None
+    last: datetime | None = None  # time of the last line read
+
+    def close(run: Run, end: datetime) -> None:
+        run.end = end
+        if run.result == "building":
+            run.result = "stopped"
+
+    for line in log.splitlines():
+        m = _EVENT.match(line)
+        if not m:
+            continue
+        when = utc(m["time"])
+        if m["processing"]:  # a new slapgrid run: one still open was killed
+            if current:
+                assert last is not None  # an open run has had at least its first line
+                close(current, last)
+            current = None
+        elif m["release"]:
+            if current:  # the previous release of the same slapgrid run
+                close(current, when)
+            current = Run(m["release"], when)
+            runs.append(current)
+        elif current is None:
+            pass
+        elif m["part"]:
+            current.parts.append((m["part"], when))
+        elif m["available"]:
+            current.result = "completed"
+        elif m["failed"]:
+            current.result = "failed"
+        elif m["finished"]:
+            close(current, when)
+            current = None
+        last = when
+    return runs
+
+
+@dataclass(frozen=True)
+class NodeUsage:
+    now: datetime
+    cpus: int
+    memory_mib: int
+    makeflags: str | None
+    disk_kib: tuple[int, int]  # (used, size) of the filesystem of /opt/slapgrid
+    sizes_kib: dict[str, int]  # path -> size, for each release dir and each shared part
+
+
+def gather(node: Node) -> tuple[list[Run], NodeUsage]:
+    """One SSH round trip: the build lines of the software logs (rotated ones first), then sizes."""
+    output = node.run(
+        "; ".join([
+            f"cd {LOG_DIR}",
+            f"cat $(ls slapos-node-software.log-* 2>/dev/null) slapos-node-software.log | grep -E '{_GREP}'",
+            f"echo {SEPARATOR}",
+            "date -u '+%Y-%m-%d %H:%M:%S'; nproc; free -m | awk '/^Mem/ {print $2}'",
+            f"grep -h '^MAKEFLAGS=' {CRON} || echo",
+            f"df -k --output=used,size {SOFTWARE_ROOT} | tail -1",
+            f"echo {SEPARATOR}",
+            f"du -sk {SOFTWARE_ROOT}/[0-9a-f]* {SHARED}/* 2>/dev/null || true",  # release dirs: md5 hex names
+        ])
+    )
+    log, facts, sizes = output.split(SEPARATOR + "\n")
+    now, cpus, memory, makeflags, disk = facts.splitlines()
+    used, size = disk.split()
+    usage = NodeUsage(
+        utc(now),
+        int(cpus),
+        int(memory),
+        makeflags.removeprefix("MAKEFLAGS=") or None,
+        (int(used), int(size)),
+        {
+            path: int(kib)
+            for kib, path in (line.split("\t") for line in sizes.splitlines())
+        },
+    )
+    return parse_runs(log), usage
+
+
+def duration(seconds: float) -> str:
+    minutes = round(seconds / 60)
+    hours, minutes = divmod(minutes, 60)
+    return f"{hours} h {minutes:02d} min" if hours else f"{minutes} min"
+
+
+def size(kib: int) -> str:
+    return f"{kib / 1024**2:.1f} GiB" if kib >= 1024**2 else f"{kib / 1024:.0f} MiB"
+
+
+def markdown(config: Config, sw: Software, runs: list[Run], usage: NodeUsage) -> str:
+    ours = [r for r in runs if sw.ref_of(r.url) is not None]
+    out = [f"# Build of {sw.name} on {config.node.title}", ""]
+    out.append(
+        f"Node: {usage.cpus} vCPU, {usage.memory_mib / 1024:.1f} GiB of memory,"
+        f" make jobs now: {usage.makeflags or 'one (no MAKEFLAGS)'} (a run keeps the setting it started with)."
+        f" Read on {usage.now:%Y-%m-%d %H:%M} UTC."
+    )
+    if not ours:
+        out += ["", f"No build of {sw.name} in the node's software logs."]
+    for url in dict.fromkeys(r.url for r in ours):
+        url_runs = [r for r in ours if r.url == url]
+        out += ["", f"## {config.url_name(url)}", "", f"<{url}>", ""]
+        out += [
+            "| Run | Started (UTC) | Ended (UTC) | Duration | Result |",
+            "|---|---|---|---|---|",
+        ]
+        for i, r in enumerate(url_runs, 1):
+            end = f"{r.end:%Y-%m-%d %H:%M}" if r.end else "running"
+            spent = duration(((r.end or usage.now) - r.start).total_seconds())
+            out.append(
+                f"| {i} | {r.start:%Y-%m-%d %H:%M} | {end} | {spent} | {r.result} |"
+            )
+        total = sum(((r.end or usage.now) - r.start).total_seconds() for r in url_runs)
+        out += ["", f"Total build time: {duration(total)} over {len(url_runs)} run(s)."]
+        longest = max(url_runs, key=lambda r: len(r.parts))
+        times = sorted(longest.part_times(usage.now), key=lambda p: -p[1])
+        if times:
+            number = url_runs.index(longest) + 1
+            out += [
+                "",
+                f'Longest parts of run {number} (from buildout\'s "Installing" lines):',
+                "",
+            ]
+            out += ["| Part | Time |", "|---|---|"]
+            out += [f"| {name} | {duration(t)} |" for name, t in times[:LONGEST]]
+            if rest := times[LONGEST:]:
+                out.append(
+                    f"| {len(rest)} other parts | {duration(sum(t for _, t in rest))} |"
+                )
+    shared = {p: k for p, k in usage.sizes_kib.items() if p.startswith(SHARED + "/")}
+    largest = sorted(shared.items(), key=lambda p: -p[1])[:LARGEST]
+    out += ["", "## Disk", "", "| What | Size |", "|---|---|"]
+    for url in dict.fromkeys(r.url for r in ours):
+        if (kib := usage.sizes_kib.get(software_dir(url))) is not None:
+            out.append(f"| {config.url_name(url)}: release directory | {size(kib)} |")
+    out.append(
+        f"| Shared parts, for every release on the node ({len(shared)}) | {size(sum(shared.values()))} |"
+    )
+    out += [
+        f"| &nbsp;&nbsp;{p.removeprefix(SHARED + '/')} | {size(k)} |"
+        for p, k in largest
+    ]
+    used, total = usage.disk_kib
+    out.append(f"| Node disk used (all of it) | {size(used)} of {size(total)} |")
+    return "\n".join(out)

hox_cli/status.py

@@ -0,0 +1,361 @@
+"""`hox status`: what the master, the node and GitHub say, about one software or the whole system."""
+
+import dataclasses
+import re
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+
+import httpx2
+
+from .config import Config, Software, short_ref
+from .master import Api, MasterError, quoted
+from .node import Node, NodeError, NodeFacts, Partition, System, git
+
+
+@dataclass(frozen=True)
+class Frontend:
+    title: str
+    allocated: bool
+
+
+@dataclass(frozen=True)
+class Instance:
+    title: str
+    state: str
+    url: str
+    software_type: str
+    partition: str | None
+    connection: dict[str, str]
+    news: str | None
+    frontends: list[Frontend]
+
+
+@dataclass(frozen=True)
+class Installation:
+    url: str
+    state: str
+
+
+@dataclass(frozen=True)
+class GitHub:
+    local_head: str
+    pushed: bool  # origin/<branch> (as last fetched) is at the local head
+    served: (
+        bool | None
+    )  # raw.githubusercontent.com serves software.cfg as at the local head; None: fetch failed
+
+
+@dataclass
+class Report:
+    software: str
+    instance: Instance | None
+    diagnosis: list[str] = field(default_factory=list)
+    installations: list[Installation] = field(default_factory=list)
+    node: NodeFacts | None = None
+    services: Partition | None = None
+    node_error: str | None = None
+    github: GitHub | None = None
+
+
+@dataclass(frozen=True)
+class Deployed:
+    """An instance tree of the project. `state` is what was requested: only the node knows what runs."""
+
+    instance: str
+    software: str | None  # its [software.<name>] of hox.toml
+    release: str  # the branch or commit of a release hox.toml knows, else its URL
+    state: str
+    partition: str | None
+    frontend: str | None
+    status: str = "unknown"
+    down: tuple[str, ...] = ()
+    memory_mb: int | None = None
+
+
+@dataclass
+class Overview:
+    instances: list[Deployed]
+    undeployed: list[str]  # software of hox.toml with no instance
+    system: System | None = None
+    facts: NodeFacts | None = None
+    node_error: str | None = None
+
+
+def overview(config: Config, api: Api, node: Node, node_reference: str) -> Overview:
+    rows = deployed(config, api)
+    result = Overview(rows, sorted(set(config.software) - {r.software for r in rows}))
+    wanted = [
+        i.url
+        for i in installations(api, node_reference)
+        if i.state == "start_requested"
+    ]
+    try:
+        result.system = node.system()
+        result.facts = node.facts(wanted)
+    except NodeError as e:
+        result.node_error = str(e)
+    result.instances = [
+        _observed(r, config.software[r.software] if r.software else None, result.system)
+        for r in rows
+    ]
+    return result
+
+
+def _observed(row: Deployed, sw: Software | None, system: System | None) -> Deployed:
+    """The row, with what runs as far as we can tell: the master only knows what was requested."""
+    if row.state != "start_requested":
+        status = {"stop_requested": "stopped", "destroy_requested": "being destroyed"}[
+            row.state
+        ]
+        return dataclasses.replace(row, status=status)
+    if row.partition is None:
+        return dataclasses.replace(row, status="not allocated")
+    if system is None:
+        return row
+    partition = _services(system, row.partition, sw)
+    if partition is None or not (partition.running or partition.down):
+        return dataclasses.replace(row, status="no services")
+    down = tuple(partition.down)
+    status = (
+        "not running"
+        if not partition.running
+        else f"{len(down)} down"
+        if down
+        else "up"
+    )
+    return dataclasses.replace(
+        row, status=status, down=down, memory_mb=partition.memory_mb
+    )
+
+
+def _services(system: System, name: str, sw: Software | None) -> Partition | None:
+    """The node's partition, leaving out of `down` the software's one-shot services."""
+    partition = next((p for p in system.partitions if p.name == name), None)
+    if partition is None or sw is None:
+        return partition
+    return dataclasses.replace(
+        partition, down=[d for d in partition.down if d not in sw.one_shot]
+    )
+
+
+# The master keeps a destroyed instance's rows: more than the project's partitions.
+LIST_LIMIT = 1000
+
+
+def deployed(config: Config, api: Api) -> list[Deployed]:
+    project = f"validation_state:validated AND follow_up_reference:={quoted(config.project.reference)}"
+    trees = api.search(
+        f'portal_type:"Instance Tree" AND {project}',
+        ["title", "slap_state", "url_string"],
+        LIST_LIMIT,
+    )
+    # Only `url` of the connection parameters: others can hold credentials (monitor-setup-url).
+    roots = {
+        r["title"]: r
+        for r in api.search(
+            f'portal_type:"Software Instance" AND {project}',
+            ["title", "aggregate_title", "getConnectionXmlAsDict"],
+            LIST_LIMIT,
+        )
+    }
+    known = {sw.instance: sw for sw in config.software.values()}
+    rows = []
+    for tree in sorted(trees, key=lambda t: t["title"]):
+        title, sw = tree["title"], known.get(tree["title"])
+        root = roots.get(title, {})
+        rows.append(
+            Deployed(
+                instance=title,
+                software=sw.name if sw else None,
+                release=_release(sw, tree["url_string"]),
+                state=tree["slap_state"],
+                partition=root.get("aggregate_title"),
+                frontend=(root.get("getConnectionXmlAsDict") or {}).get("url"),
+            )
+        )
+    return rows
+
+
+def _release(sw: Software | None, url: str) -> str:
+    ref = sw.ref_of(url) if sw else None
+    return url if ref is None else short_ref(ref)
+
+
+def gather(
+    config: Config,
+    sw: Software,
+    api: Api,
+    node: Node,
+    http: httpx2.Client,
+    node_reference: str,
+) -> Report:
+    report = Report(sw.name, instance(api, sw.instance))
+    if report.instance and report.instance.partition is None:
+        report.diagnosis = allocation_diagnosis(api, config, sw, report.instance)
+    report.installations = [
+        i for i in installations(api, node_reference) if sw.ref_of(i.url) is not None
+    ]
+    try:
+        report.node = node.facts([i.url for i in report.installations])
+        if report.instance and (partition := report.instance.partition):
+            report.services = _services(node.system(), partition, sw)
+            promises = [
+                p for p in report.node.failing_promises if p.startswith(f"{partition}[")
+            ]
+            report.node = dataclasses.replace(report.node, failing_promises=promises)
+    except NodeError as e:
+        report.node_error = str(e)
+    report.github = github(sw, http)
+    return report
+
+
+def instance(api: Api, title: str) -> Instance | None:
+    trees = api.search(
+        f'portal_type:"Instance Tree" AND validation_state:validated AND title:={quoted(title)}',
+        ["title", "relative_url", "slap_state", "url_string", "source_reference"],
+    )
+    if not trees:
+        return None
+    tree = trees[0]
+    rows = api.search(
+        'portal_type:("Software Instance","Slave Instance") AND validation_state:validated'
+        f" AND default_specialise_title:={quoted(title)}",
+        [
+            "title",
+            "portal_type",
+            "slap_state",
+            "aggregate_title",
+            "getConnectionXmlAsDict",
+            "SoftwareInstance_getNewsDict",
+        ],
+    )
+    root = next(
+        (
+            r
+            for r in rows
+            if r["portal_type"] == "Software Instance" and r["title"] == title
+        ),
+        {},  # none yet: not allocated
+    )
+    frontends = [
+        Frontend(r["title"], r["aggregate_title"] is not None)
+        for r in rows
+        if r["portal_type"] == "Slave Instance"
+    ]
+    news: dict = root.get("SoftwareInstance_getNewsDict") or {}
+    return Instance(
+        title=title,
+        state=tree["slap_state"],
+        url=tree["url_string"],
+        software_type=tree["source_reference"],
+        partition=root.get("aggregate_title"),
+        connection=root.get("getConnectionXmlAsDict") or {},
+        news=news.get("text") if not news.get("no_data") else None,
+        frontends=frontends,
+    )
+
+
+def require_instance(api: Api, sw: Software) -> Instance:
+    """The software's instance; an error that says what to do when the master has none."""
+    inst = instance(api, sw.instance)
+    if inst is None:
+        raise MasterError(f"no instance {sw.instance!r} on the master (`hox request`)")
+    return inst
+
+
+def installations(api: Api, node_reference: str) -> list[Installation]:
+    rows = api.search(
+        'portal_type:"Software Installation" AND validation_state:validated'
+        f" AND default_aggregate_reference:={quoted(node_reference)}",
+        ["url_string", "slap_state"],
+    )
+    return [Installation(r["url_string"], r["slap_state"]) for r in rows]
+
+
+def allocation_diagnosis(
+    api: Api, config: Config, sw: Software, inst: Instance
+) -> list[str]:
+    """Why an instance has no partition: what the project lacks for its URL and type."""
+    if inst.software_type != sw.software_type:
+        return [
+            f"instance software type is {inst.software_type!r}, config says {sw.software_type!r}"
+        ]
+    releases = api.search(
+        f'portal_type:"Software Product Release Variation" AND url_string:={quoted(inst.url)}',
+        ["relative_url"],
+    )
+    if not releases:
+        return [
+            f"{inst.url} is not a release of any Software Product: `hox setup` or `hox release`"
+        ]
+    supplies = api.search(
+        'portal_type:"Allocation Supply" AND validation_state:validated'
+        f" AND destination_project_reference:={quoted(config.project.reference)}",
+        ["relative_url", "title"],
+    )
+    if not supplies:
+        return [f"no validated Allocation Supply in {config.project.reference}"]
+    lines = api.search(
+        f'portal_type:"Allocation Supply Line" AND resource_title:={quoted(sw.product)}',
+        ["relative_url"],
+    )
+    for supply in supplies:
+        for line in lines:
+            if line["relative_url"].startswith(supply["relative_url"] + "/"):
+                if (inst.url, inst.software_type) in ticked_cells(
+                    api.get(line["relative_url"], "view")
+                ):
+                    return [
+                        f"allocatable through {supply['title']!r}: wait for the master's allocation pass"
+                    ]
+    return [f"no validated Allocation Supply ticks ({inst.url}, {inst.software_type})"]
+
+
+_CELL = re.compile(r"_cell_\d+_(\d+)_\d+$")
+ALLOCABLE = "matrixbox_allocable"
+
+
+def matrix_cells(line_view: dict) -> Iterator[tuple[str, str, dict]]:
+    """(release URL, software type, field) for each field of an Allocation Supply Line's matrix.
+
+    The matrix is a list of tables (one per value of a third dimension, unused
+    here), each a header row of software types and one row per release. A cell
+    has three fields: the `allocable` checkbox and two hidden category lists.
+    """
+    for table in line_view["_embedded"]["_view"]["matrixbox"]["data"]:
+        header, rows = table[0], table[1:]
+        for row in rows:
+            for field in row[1:]:
+                cell = _CELL.search(field["key"])
+                if cell is None:
+                    raise MasterError(
+                        f"Allocation Supply matrix: unexpected field {field['key']!r}"
+                    )
+                column = int(cell.group(1))
+                yield _label(row[0]), _label(header[1 + column]), field
+
+
+def ticked_cells(line_view: dict) -> set[tuple[str, str]]:
+    """(release URL, software type) pairs ticked in an Allocation Supply Line's matrix."""
+    return {
+        (r, t)
+        for r, t, f in matrix_cells(line_view)
+        if f["field_id"] == ALLOCABLE and f["value"]
+    }
+
+
+def _label(label: str) -> str:
+    # "Software Release/https://…" -> "https://…"
+    return label.split("/", 1)[1]
+
+
+def github(sw: Software, http: httpx2.Client) -> GitHub:
+    head = git(sw.checkout, "rev-parse", sw.branch).strip()
+    remote = git(sw.checkout, "rev-parse", f"origin/{sw.branch}").strip()
+    expected = git(sw.checkout, "show", f"{head}:{sw.path}")
+    try:
+        response = http.get(sw.branch_url)
+        served = response.status_code == 200 and response.text == expected
+    except httpx2.TransportError:
+        served = None
+    return GitHub(head, head == remote, served)