nodeview-slurm 0.1.0__py3-none-any.whl

nodeview/snapshot.py

@@ -0,0 +1,72 @@
+"""Static snapshot of the current state, for --once.
+
+Handy when you don't want an interactive interface: inside a script, in a
+pipe, in a .bashrc, or on a terminal too small for the TUI.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+
+from rich.console import Console
+from rich.panel import Panel
+
+from . import analysis as an
+from . import config
+from . import i18n
+from .i18n import t
+from . import palette
+from . import render
+from .source import fetch
+
+
+def _panel(content, title: str) -> Panel:
+    return Panel(content, title=f"[bold cyan]{title}[/]", title_align="left",
+                 border_style="cyan", padding=(0, 1))
+
+
+def _saved_palette():
+    """The static snapshot also follows the theme chosen in the TUI: if you
+    picked a light theme, printing with the dark palette would be inconsistent."""
+    name = config.load().get('theme')
+    if isinstance(name, str):
+        try:
+            from textual.theme import BUILTIN_THEMES
+            theme = BUILTIN_THEMES.get(name)
+            if theme is not None:
+                return palette.from_theme(theme)
+        except Exception:
+            pass
+    return palette.default()
+
+
+def print_once(width=None, cluster=False) -> int:
+    # the static snapshot also follows the language chosen in the TUI
+    lang = config.load().get('lang')
+    if isinstance(lang, str):
+        i18n.use(lang)
+    palette.use(_saved_palette())
+    console = Console(width=width)
+    try:
+        snap = fetch()
+    except Exception as exc:
+        console.print(f"[bold red]{t('once.error')}[/] {exc}")
+        return 1
+
+    when = datetime.fromtimestamp(snap.taken_at).strftime('%H:%M:%S')
+    console.print("[bold]nodeview[/]  [dim]"
+                  + t('once.header', me=snap.me, nodes=len(snap.nodes), when=when,
+                      secs=f"{snap.duration:.2f}") + "[/]")
+    if cluster:
+        occupants = palette.Occupants(an.occupant_counts(snap), me=snap.me)
+        console.print(_panel(render.totals_strip(snap), t('card.totals')))
+        console.print(_panel(render.node_table(snap, occupants), t('card.nodes')))
+        console.print(_panel(render.breakdown(snap, occupants), t('card.usage')))
+        console.print(render.legend(occupants))
+        console.print(_panel(render.specs_table(snap), t('card.specs')))
+    else:
+        console.print(_panel(render.my_jobs(snap), t('card.my_jobs')))
+        console.print(_panel(render.where_do_i_fit(snap), t('card.fit')))
+        console.print(_panel(render.cluster(snap), t('card.cluster')))
+        from .profile import read_profile
+        console.print(_panel(render.my_account(read_profile()), t('card.account')))
+    return 0

nodeview/source.py

@@ -0,0 +1,306 @@
+"""Reading Slurm state.
+
+Two sources, each for what it does best:
+
+- `scontrol -d -o show node` (in nodes.py) for nodes: gives everything the
+  cluster exposes, including the actual GPU model, the indices of busy GPUs,
+  allocated RAM versus truly free RAM, and CPU load;
+- **nodeocc**'s readers for jobs, reservations and QOS limits, which already
+  know how to interpret this cluster's quirks.
+
+The result is our own dataclasses: the rest of the app never touches
+nodeocc's internal classes.
+"""
+from __future__ import annotations
+
+import math
+import os
+import pwd
+import time
+from dataclasses import dataclass, field
+from typing import Optional
+
+from .i18n import t
+from .nodes import NodeSpec, read_gpu_owners, read_nodes, short_name
+
+
+class SourceUnavailable(RuntimeError):
+    """nodeocc is not installed or cannot be imported."""
+
+
+class _ReaderShim:
+    """Silent stand-in for nodeocc's Singleton.
+
+    The readers only use the instance for err()/log()/timeme(), but the real
+    constructor also runs master election, opens UDP sockets and writes
+    portfiles in /tmp. None of that is needed here, and joining in would
+    disturb an already running nodeocc: we install this in its place.
+    """
+
+    def __getattr__(self, name):
+        return lambda *args, **kwargs: None
+
+
+def _ensure_readers():
+    try:
+        from view.curses_multiwindow import Singleton
+    except ImportError as exc:  # pragma: no cover
+        from .i18n import t
+        raise SourceUnavailable(t('src.missing')) from exc
+
+    if Singleton.getInstance(create=False) is None:
+        # access via the mangled name: the only way to register the instance
+        # without running __init__ and its side effects
+        Singleton._Singleton__instance = _ReaderShim()
+
+    import readers.slurmreader as slurmreader
+    return slurmreader
+
+
+def _clean(value, default=None):
+    """pandas nan and empty strings -> None."""
+    if value is None:
+        return default
+    if isinstance(value, float) and math.isnan(value):
+        return default
+    if isinstance(value, str) and not value.strip():
+        return default
+    return value
+
+
+@dataclass(frozen=True)
+class Occupancy:
+    """The slice of a node taken by a job."""
+    jobid: str
+    job_name: str
+    user: str
+    group: Optional[str]
+    account: Optional[str]
+    partition: Optional[str]
+    qos: Optional[str]
+    gpus: int
+    cpus: int
+    mem_mb: float
+    mine: bool
+
+
+@dataclass
+class Node(NodeSpec):
+    """A node: Slurm's specs plus who is running on it."""
+    reserved: str = 'no'            # 'no' | 'yes' | 'pending'
+    occupants: tuple = ()
+    gpu_owners: dict = field(default_factory=dict)   # GPU index -> user
+
+    @property
+    def status(self) -> str:
+        """'ok' | 'drain' | 'down', the classification used across the app."""
+        state = self.state.upper()
+        if any(b in state for b in ('DOWN', 'FAIL', 'INVAL', 'UNKNOWN', 'NOT_RESPONDING')):
+            return 'down'
+        if 'DRAIN' in state or 'MAINT' in state:
+            return 'drain'
+        return 'ok'
+
+    @property
+    def healthy(self) -> bool:
+        return self.status == 'ok'
+
+    @property
+    def schedulable(self) -> bool:
+        return self.healthy and self.reserved == 'no' and self.gpus_free > 0
+
+    # NB: NodeSpec.mem_free_mb is the RAM the OS reports as free; to decide
+    # where a job fits, what matters is mem_unalloc_mb, i.e. what Slurm hasn't
+    # assigned yet. A job may have reserved memory without touching it yet,
+    # and the two figures diverge a lot.
+
+    @property
+    def users(self) -> tuple:
+        seen = []
+        for occ in self.occupants:
+            if occ.user not in seen:
+                seen.append(occ.user)
+        return tuple(seen)
+
+
+@dataclass(frozen=True)
+class Job:
+    jobid: str
+    name: str
+    user: str
+    group: Optional[str]
+    state: str
+    runtime: str
+    reason: Optional[str]
+    partition: str
+    account: Optional[str]
+    qos: Optional[str]
+    priority: Optional[int]
+    gpus: int
+    cpus: int
+    mem_mb: float
+    nodes: tuple
+    mine: bool
+
+    @property
+    def running(self) -> bool:
+        return self.state in ('R', 'CG')
+
+    @property
+    def pending(self) -> bool:
+        return not self.running
+
+
+@dataclass(frozen=True)
+class Maintenance:
+    nodes: tuple
+    start: str
+    end: str
+
+
+@dataclass
+class Snapshot:
+    nodes: list
+    jobs: list
+    gpu_limit_user: Optional[float]
+    gpu_limit_group: Optional[float]
+    maintenances: list
+    me: str
+    taken_at: float
+    duration: float = 0.0
+
+    @property
+    def my_jobs(self) -> list:
+        return [j for j in self.jobs if j.mine]
+
+    @property
+    def running_jobs(self) -> list:
+        return [j for j in self.jobs if j.running]
+
+    @property
+    def pending_jobs(self) -> list:
+        return [j for j in self.jobs if j.pending]
+
+    @property
+    def healthy_nodes(self) -> list:
+        return [n for n in self.nodes if n.healthy]
+
+    @property
+    def gpus_total(self) -> int:
+        return sum(n.gpus_total for n in self.healthy_nodes)
+
+    @property
+    def gpus_used(self) -> int:
+        return sum(n.gpus_used for n in self.nodes)
+
+    @property
+    def gpus_free(self) -> int:
+        return sum(n.gpus_free for n in self.nodes if n.schedulable)
+
+    @property
+    def cpus_total(self) -> int:
+        return sum(n.cpus_total for n in self.healthy_nodes)
+
+    @property
+    def cpus_used(self) -> int:
+        return sum(n.cpus_alloc for n in self.nodes)
+
+    @property
+    def mem_total_mb(self) -> float:
+        return sum(n.mem_total_mb for n in self.healthy_nodes)
+
+    @property
+    def mem_used_mb(self) -> float:
+        return sum(n.mem_alloc_mb for n in self.nodes)
+
+    @property
+    def unhealthy_nodes(self) -> list:
+        return [n for n in self.nodes if not n.healthy]
+
+
+def current_user() -> str:
+    return os.environ.get('USER') or pwd.getpwuid(os.getuid()).pw_name
+
+
+def _reserved_map(inf) -> dict:
+    return {n.name: (n.reserved or 'no') for n in getattr(inf, 'nodes', []) or []}
+
+
+def fetch() -> Snapshot:
+    """A full read of the cluster. It takes a few tenths of a second and is
+    meant to run in a worker thread."""
+    slurmreader = _ensure_readers()
+    me = current_user()
+    started = time.perf_counter()
+
+    specs = read_nodes()
+    owners = read_gpu_owners()
+    inf = slurmreader.read_infrastructure()
+    raw_jobs, _ = slurmreader.read_jobs()
+
+    reserved = _reserved_map(inf)
+
+    jobs = []
+    per_node = {}
+    for j in raw_jobs:
+        job = Job(
+            jobid=str(j.jobid),
+            name=str(_clean(j.name, t('fallback.job_name'))),
+            user=str(j.user),
+            group=_clean(j.user_group),
+            state=str(j.state),
+            runtime=str(_clean(j.runtime, '-')),
+            reason=_clean(j.reason),
+            partition=str(_clean(j.partition, '-')),
+            account=_clean(j.account),
+            qos=None,
+            priority=int(j.priority) if _clean(j.priority) is not None else None,
+            gpus=sum(int(jl.n_gpus or 0) for jl in j.joblets),
+            cpus=sum(int(jl.cpus or 0) for jl in j.joblets),
+            mem_mb=sum(float(jl.mem or 0) for jl in j.joblets),
+            nodes=tuple(jl.node for jl in j.joblets if jl.node),
+            mine=(j.user == me),
+        )
+        jobs.append(job)
+
+        if job.running:
+            for jl in j.joblets:
+                if not jl.node:
+                    continue
+                per_node.setdefault(jl.node, []).append(Occupancy(
+                    jobid=job.jobid,
+                    job_name=job.name,
+                    user=job.user,
+                    group=job.group,
+                    account=job.account,
+                    partition=job.partition,
+                    qos=job.qos,
+                    gpus=int(jl.n_gpus or 0),
+                    cpus=int(jl.cpus or 0),
+                    mem_mb=float(jl.mem or 0),
+                    mine=job.mine,
+                ))
+
+    nodes = []
+    for spec in specs:
+        occ = tuple(sorted(per_node.get(spec.name, []),
+                           key=lambda o: (-o.gpus, o.user, o.jobid)))
+        nodes.append(Node(**vars(spec), reserved=reserved.get(spec.name, 'no'),
+                          occupants=occ, gpu_owners=owners.get(spec.name, {})))
+
+    maintenances = [
+        Maintenance(tuple(short_name(n) for n in (m.nodes or ())),
+                    str(m.start_time), str(m.end_time))
+        for m in (getattr(inf, 'maintenances', None) or [])
+    ]
+
+    return Snapshot(
+        nodes=nodes,
+        jobs=jobs,
+        gpu_limit_user=_clean(inf.gpu_limit_pu),
+        gpu_limit_group=_clean(inf.gpu_limit_grp),
+        maintenances=maintenances,
+        me=me,
+        taken_at=time.time(),
+        duration=time.perf_counter() - started,
+    )

nodeview_slurm-0.1.0.dist-info/METADATA

@@ -0,0 +1,41 @@
+Metadata-Version: 2.5
+Name: nodeview-slurm
+Version: 0.1.0
+Summary: GPU status of a Slurm cluster, from the user's point of view.
+Project-URL: Homepage, https://codeberg.org/danielrossi/NodeView
+Project-URL: Issues, https://codeberg.org/danielrossi/NodeView/issues
+Author-email: Daniel Rossi <daniel.rossi@unimore.it>
+Keywords: gpu,hpc,slurm,textual,tui
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: System :: Monitoring
+Requires-Python: >=3.9
+Requires-Dist: nodeocc<1.1,>=1.0.26
+Requires-Dist: rich>=13
+Requires-Dist: textual>=8.0
+Description-Content-Type: text/markdown
+
+# NodeView
+
+GPU status of a Slurm cluster, from the user's point of view: your GPUs, the
+cluster's nodes and the queue, in a terminal interface.
+
+## Installation
+
+```
+python3 -m pip install nodeview-slurm
+```
+
+## Usage
+
+```
+nodeview                 # interactive interface
+nodeview --once          # print a snapshot and exit
+nodeview --once --cluster
+nodeview --interval 5    # refresh the state every 5 seconds
+```
+
+Requires Python 3.9+ and the Slurm commands (`scontrol`, `squeue`, `sinfo`,
+`sacct`, ...) on your `PATH`.

nodeview_slurm-0.1.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+nodeview/__init__.py,sha256=FAAjqB48n7gij9f-r4wOtF3sgZUJK7E-ezpr72mvg2U,101
+nodeview/__main__.py,sha256=kXRknv5vG_c8Fqw8MFWJKNpVkib70WxXBbnQ2Tb-OVU,1219
+nodeview/analysis.py,sha256=rBo-Mg0S9XjeIuPAl8bqABTs4XA4lWdeoJ5qiyCRHL8,9997
+nodeview/app.py,sha256=q7QYboR0DmBwJCrPC3AGXVhX0jriMktOTi7wnRoC3og,15750
+nodeview/app.tcss,sha256=UdlfEBOcFFIyFQ2M-1UBErtSX10le3xYZjxFLztq940,601
+nodeview/config.py,sha256=ybg8NB2RDOFI3ufK1wEis80rJhXT1SqnqPAchnH2s08,1869
+nodeview/i18n.py,sha256=Wb40MSJdhMoDXX6g_XpV4JDqul4ba47ulSrADIc8CT8,18887
+nodeview/manual.py,sha256=Ovm93ateDtaYSQxFvxrPKsRDfFSkfAYMWx2-BbFf2Aw,6901
+nodeview/nodes.py,sha256=IuRUkuBeqcaEuhFVMhVneMcqUhhiZLzkXp0blVytdDA,10668
+nodeview/palette.py,sha256=_GVZxJbVnYpuOdFUtGQ1jyITKNmorlUBP1bmyp_J0t0,11400
+nodeview/profile.py,sha256=ni8wxL8Ak1DmyBJOd4CnULkWFSzAQkfvGtvFU2_lols,9453
+nodeview/render.py,sha256=VSAodSvdJYYLPe1mBBVqtOBlHHU6LS1yO39nODFRq5c,27358
+nodeview/snapshot.py,sha256=TFlgHojQwlfFSP_hjTyCvUsS_D-w3_5tFDZBJkzvGik,2672
+nodeview/source.py,sha256=9TEfaZ8zYgIfpTulABKkVKiYSN5K_fwUM3n7yhD5u2k,8786
+nodeview_slurm-0.1.0.dist-info/METADATA,sha256=FSPhUZmHZn-8LMNkmrdpam3aZH8jD3MrtlJqmCffmL0,1251
+nodeview_slurm-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+nodeview_slurm-0.1.0.dist-info/entry_points.txt,sha256=T-Is8Gv-vdgiGkbE36usLI3DxS0lD2dlGqS8gXhg9uI,52
+nodeview_slurm-0.1.0.dist-info/RECORD,,

nodeview_slurm-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

nodeview_slurm-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nodeview = nodeview.__main__:main