nodeview-slurm 0.1.0__py3-none-any.whl

nodeview/__init__.py

@@ -0,0 +1,3 @@
+"""NodeView: GPU status of a Slurm cluster, from the user's point of view."""
+
+__version__ = "0.1.0"

nodeview/__main__.py

@@ -0,0 +1,32 @@
+"""Entry point: TUI by default, static snapshot with --once."""
+from __future__ import annotations
+
+import argparse
+import sys
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog='nodeview',
+        description="GPU status of a Slurm cluster, from the user's point of view.")
+    parser.add_argument('--once', action='store_true',
+                        help="print a snapshot and exit, without the interactive interface")
+    parser.add_argument('--interval', type=float, default=10.0, metavar='SEC',
+                        help="how often to refresh the state (default: 10s)")
+    parser.add_argument('--cluster', action='store_true',
+                        help="with --once, print the cluster page instead of yours")
+    parser.add_argument('--width', type=int, default=None, metavar='N',
+                        help="force the output width for --once (useful in pipes and scripts)")
+    args = parser.parse_args(argv)
+
+    if args.once:
+        from .snapshot import print_once
+        return print_once(width=args.width, cluster=args.cluster)
+
+    from .app import NodeView
+    NodeView(interval=args.interval).run()
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

nodeview/analysis.py

@@ -0,0 +1,298 @@
+"""Everything that can be derived from a Snapshot.
+
+Deliberately kept apart from the UI: the same functions serve both the TUI
+and the static --once snapshot.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
+
+from .i18n import t
+from .source import Node, Snapshot
+
+# Slurm's codes are cryptic, and they're the first thing you look at when a
+# job won't start. The texts live in the language catalog; only codes here.
+REASON_CODES = (
+    'QOSGrpGRES', 'QOSGrpGRESMinutes', 'QOSGrpMemLimit', 'QOSGrpCpuLimit',
+    'QOSMaxGRESPerUser', 'QOSMaxCpuPerUserLimit', 'QOSMaxMemPerUser',
+    'QOSMaxJobsPerUserLimit', 'Dependency', 'Resources', 'Priority',
+    'BeginTime', 'ReqNodeNotAvail', 'JobArrayTaskLimit',
+)
+
+
+def _reason_texts(code: str) -> tuple:
+    return (t(f'reason.{code}.label'), t(f'reason.{code}.detail'))
+
+
+def explain_reason(reason: Optional[str]) -> tuple:
+    """(short label, explanation) for a Slurm pending reason."""
+    if not reason or reason.strip() in ('None', 'none'):
+        return _reason_texts('unknown')
+    key = reason.split(',')[0].strip()
+    if key in REASON_CODES:
+        return _reason_texts(key)
+    for code in REASON_CODES:
+        if key.startswith(code):
+            return _reason_texts(code)
+    return (reason, "")
+
+
+@dataclass
+class Fit:
+    """Answers: if I ask for N GPUs, where do I fit right now?"""
+    gpus: int
+    nodes: list
+    over_user_quota: bool
+
+    @property
+    def possible(self) -> bool:
+        return bool(self.nodes) and not self.over_user_quota
+
+
+@dataclass
+class Quota:
+    used: int
+    limit: Optional[float]
+
+    @property
+    def headroom(self) -> Optional[int]:
+        if self.limit is None:
+            return None
+        return max(0, int(self.limit) - self.used)
+
+    @property
+    def ratio(self) -> float:
+        if not self.limit:
+            return 0.0
+        return min(1.0, self.used / self.limit)
+
+
+def my_gpus(snap: Snapshot) -> int:
+    return sum(j.gpus for j in snap.my_jobs if j.running)
+
+
+def my_quota(snap: Snapshot) -> Quota:
+    return Quota(used=my_gpus(snap), limit=snap.gpu_limit_user)
+
+
+def gpus_by_group(snap: Snapshot) -> list:
+    """GPUs in use per unix group, from the hungriest to the least."""
+    totals = {}
+    for job in snap.running_jobs:
+        key = job.group or t('fallback.group')
+        totals[key] = totals.get(key, 0) + job.gpus
+    return sorted(totals.items(), key=lambda kv: kv[1], reverse=True)
+
+
+def gpus_by_account(snap: Snapshot) -> list:
+    """GPUs in use per Slurm account: that's where courses (cvcs) stand apart
+    from projects and theses."""
+    totals = {}
+    for job in snap.running_jobs:
+        key = job.account or t('fallback.account')
+        totals[key] = totals.get(key, 0) + job.gpus
+    return sorted(((k, v) for k, v in totals.items() if v),
+                  key=lambda kv: kv[1], reverse=True)
+
+
+def by_gpu_model(snap: Snapshot) -> list:
+    """(model, used, total): tells which GPU type is still free."""
+    totals = {}
+    for node in snap.nodes:
+        if not node.healthy:
+            continue
+        used, total = totals.get(node.gpu_model, (0, 0))
+        totals[node.gpu_model] = (used + node.gpus_used, total + node.gpus_total)
+    return sorted(((m, u, t) for m, (u, t) in totals.items()),
+                  key=lambda r: (-(r[2] - r[1]), r[0]))
+
+
+def by_partition(snap: Snapshot) -> list:
+    """(partition, nodes, total GPUs)."""
+    totals = {}
+    for node in snap.nodes:
+        for part in node.partitions:
+            n, g = totals.get(part, (0, 0))
+            totals[part] = (n + 1, g + node.gpus_total)
+    return sorted(((p, n, g) for p, (n, g) in totals.items()),
+                  key=lambda r: -r[2])
+
+
+def occupant_counts(snap: Snapshot) -> dict:
+    """Busy GPUs per user: this is the input to the identity palette."""
+    counts = {}
+    for node in snap.nodes:
+        for occ in node.occupants:
+            counts[occ.user] = counts.get(occ.user, 0) + max(occ.gpus, 0)
+    return counts
+
+
+def candidate_nodes(snap: Snapshot, gpus: int, min_mem_gb: float = 0.0) -> list:
+    """Nodes that would accept a `gpus`-GPU job right away.
+
+    It also takes at least one CPU and some free RAM per GPU: a node with free
+    GPUs but no CPUs left isn't really usable.
+    """
+    out = []
+    for node in snap.nodes:
+        if not node.schedulable or node.gpus_free < gpus:
+            continue
+        if node.cpus_free < gpus:
+            continue
+        if node.mem_unalloc_mb < max(min_mem_gb * 1024, 1024 * gpus):
+            continue
+        out.append(node)
+    return sorted(out, key=lambda n: (-n.gpus_free, -n.mem_free_mb, n.name))
+
+
+def fits(snap: Snapshot, sizes=(1, 2, 4, 8), min_mem_gb: float = 0.0) -> list:
+    quota = my_quota(snap)
+    room = quota.headroom
+    out = []
+    for size in sizes:
+        over = room is not None and size > room
+        out.append(Fit(
+            gpus=size,
+            nodes=[] if over else candidate_nodes(snap, size, min_mem_gb),
+            over_user_quota=over,
+        ))
+    return out
+
+
+def queue_reasons(snap: Snapshot) -> list:
+    """(raw reason, label, explanation, count) sorted by frequency."""
+    counts = {}
+    for job in snap.pending_jobs:
+        key = (job.reason or 'None').split(',')[0].strip()
+        counts[key] = counts.get(key, 0) + 1
+    rows = []
+    for raw, n in sorted(counts.items(), key=lambda kv: kv[1], reverse=True):
+        label, detail = explain_reason(raw)
+        rows.append((raw, label, detail, n))
+    return rows
+
+
+def headline(snap: Snapshot) -> Optional[str]:
+    """One sentence that explains the queue, when there's something to say.
+
+    The interesting case: a long queue but free GPUs. It means the bottleneck
+    is a quota, not the hardware, and nobody notices by looking at usage
+    alone.
+    """
+    pending = len(snap.pending_jobs)
+    if not pending:
+        return None
+    reasons = queue_reasons(snap)
+    if not reasons:
+        return None
+    raw, label, _, n = reasons[0]
+    free = snap.gpus_free
+    if raw.startswith('QOS') and free > 0 and n >= max(2, pending // 3):
+        return t('head.quota_not_gpus', n=n, total=pending, label=label, free=free)
+    if raw == 'Resources' and free == 0:
+        return t('head.all_busy', n=n)
+    return t('head.generic', n=n, total=pending, label=label)
+
+
+def warnings(snap: Snapshot) -> list:
+    """Warnings as (level, text). Level: 'info' | 'warn' | 'bad'."""
+    out = []
+    for status, level, key in (('down', 'bad', 'warn.down'), ('drain', 'warn', 'warn.drain')):
+        hit = [n for n in snap.nodes if n.status == status]
+        if hit:
+            noun = t('warn.node_one' if len(hit) == 1 else 'warn.node_many')
+            out.append((level, t(key, n=len(hit), noun=noun,
+                                 nodes=", ".join(n.name for n in hit))))
+
+    reserved = [n for n in snap.nodes if n.reserved != 'no' and n.healthy]
+    if reserved:
+        noun = t('warn.reserved_one' if len(reserved) == 1 else 'warn.reserved_many')
+        out.append(('info', t('warn.reserved', n=len(reserved), noun=noun,
+                              nodes=", ".join(n.name for n in reserved))))
+
+    for m in snap.maintenances:
+        out.append((maintenance_level(m), describe_maintenance(m, len(snap.nodes))))
+    return out
+
+
+def _parse_when(value: str) -> Optional[datetime]:
+    for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%dT%H:%M:%S'):
+        try:
+            return datetime.strptime(str(value), fmt)
+        except (ValueError, TypeError):
+            continue
+    return None
+
+
+def days_to_maintenance(m) -> Optional[float]:
+    start = _parse_when(m.start)
+    if start is None:
+        return None
+    return (start - datetime.now()).total_seconds() / 86400
+
+
+def maintenance_level(m) -> str:
+    days = days_to_maintenance(m)
+    if days is None:
+        return 'info'
+    if days <= 0:
+        return 'bad'
+    return 'warn' if days <= 7 else 'info'
+
+
+def describe_maintenance(m, total_nodes: int) -> str:
+    """Compact: the full node list takes three lines and says nothing."""
+    start, end = _parse_when(m.start), _parse_when(m.end)
+    n = len(m.nodes)
+    if n >= max(1, total_nodes - 1):
+        where = t('warn.whole_cluster')
+    elif n <= 4:
+        where = ", ".join(m.nodes)
+    else:
+        where = t('warn.n_nodes_sample', n=n, sample=', '.join(m.nodes[:3]))
+
+    if start is None:
+        return t('warn.maint_planned', where=where)
+
+    days = days_to_maintenance(m)
+    when = start.strftime('%d/%m %H:%M')
+    if days is not None and days <= 0:
+        quando = t('warn.maint_now')
+    elif days is not None and days < 1:
+        quando = t('warn.maint_hours', n=int(days * 24))
+    elif days is not None:
+        d = int(days)
+        quando = t('warn.maint_days', n=d,
+                   noun=t('warn.day_one' if d == 1 else 'warn.day_many'))
+    else:
+        quando = when
+
+    lasting = ""
+    if end is not None and start is not None:
+        g = max(1, round((end - start).total_seconds() / 86400))
+        lasting = t('warn.lasting', n=g,
+                    noun=t('warn.day_one' if g == 1 else 'warn.day_many'))
+    return t('warn.maint', when=quando, date=when, lasting=lasting, where=where)
+
+
+def uptime(node) -> Optional[str]:
+    """How long the node has been up, in readable form."""
+    start = _parse_when((node.boot_time or '').replace('T', ' '))
+    if start is None:
+        return None
+    days = (datetime.now() - start).days
+    if days >= 1:
+        return f"{days} " + t('warn.day_one' if days == 1 else 'warn.day_many')
+    hours = int((datetime.now() - start).total_seconds() // 3600)
+    return t('time.hours', n=hours)
+
+
+def node_sort_key(node: Node):
+    """Nodes you can fit on first, then the busiest, broken ones last."""
+    if not node.healthy:
+        return (2, 0, node.name)
+    if node.gpus_free > 0 and node.reserved == 'no':
+        return (0, -node.gpus_free, node.name)
+    return (1, -node.gpus_free, node.name)