nodeview-slurm 0.1.0__tar.gz → 0.2.0__tar.gz

{nodeview_slurm-0.1.0 → nodeview_slurm-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: nodeview-slurm
-Version: 0.1.0
+Version: 0.2.0
 Summary: GPU status of a Slurm cluster, from the user's point of view.
 Project-URL: Homepage, https://codeberg.org/danielrossi/NodeView
 Project-URL: Issues, https://codeberg.org/danielrossi/NodeView/issues

{nodeview_slurm-0.1.0 → nodeview_slurm-0.2.0}/src/nodeview/__init__.py

@@ -1,3 +1,3 @@
 """NodeView: GPU status of a Slurm cluster, from the user's point of view."""
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"

{nodeview_slurm-0.1.0 → nodeview_slurm-0.2.0}/src/nodeview/app.py

@@ -29,6 +29,10 @@ from . import palette
 from . import render
 from .profile import Profile, read_profile
 from .source import Snapshot, fetch
+from .usage import GpuUsage, read_gpu_usage
+
+# GPU hours move slowly and the call costs ~0.2 s: every few minutes is plenty
+USAGE_EVERY = 300.0
 
 
 def _retranslate(cls, bindings) -> None:
@@ -137,6 +141,8 @@ class MainScreen(DataScreen):
                 yield Static(id="fit")
             with Card(t('card.cluster'), classes="card"):
                 yield Static(id="cluster")
+            with Card(t('card.gpu_hours'), classes="card"):
+                yield Static(id="usage")
             # at the bottom on purpose: this is info you rarely look at
             with Card(t('card.account'), classes="card"):
                 yield Static(id="account")
@@ -146,8 +152,16 @@ class MainScreen(DataScreen):
         self.query_one("#mine", Static).update(render.my_jobs(snap))
         self.query_one("#fit", Static).update(render.where_do_i_fit(snap))
         self.query_one("#cluster", Static).update(render.cluster(snap))
+        self.apply_usage()
         self.apply_profile(getattr(self.app, "profile", None))
 
+    def apply_usage(self) -> None:
+        try:
+            widget = self.query_one("#usage", Static)
+        except Exception:
+            return
+        widget.update(render.gpu_usage(self.app.usage, loaded=self.app.usage_loaded))
+
     def apply_profile(self, prof) -> None:
         try:
             widget = self.query_one("#account", Static)
@@ -268,6 +282,8 @@ class NodeView(App):
         self._error: str | None = None
         self._restored = False
         self.profile: Profile | None = None
+        self.usage: GpuUsage | None = None
+        self.usage_loaded = False
 
     def on_mount(self) -> None:
         self._restore_prefs()
@@ -278,6 +294,8 @@ class NodeView(App):
         # budget, quotas and expiration cost ~1.7s and don't change minute to
         # minute: they're read only once at startup, outside the loop
         self._load_profile()
+        self._load_usage()
+        self.set_interval(USAGE_EVERY, self._load_usage)
 
     # --- data -------------------------------------------------------------
 
@@ -303,6 +321,23 @@ class NodeView(App):
             return
         self.call_from_thread(self._on_profile, prof)
 
+    @work(exclusive=True, thread=True, group="usage")
+    def _load_usage(self) -> None:
+        try:
+            usage = read_gpu_usage()
+        except Exception:
+            usage = None
+        self.call_from_thread(self._on_usage, usage)
+
+    def _on_usage(self, usage: GpuUsage | None) -> None:
+        # a failed read keeps the last good one: better slightly stale than empty
+        if usage is not None or not self.usage_loaded:
+            self.usage = usage
+        self.usage_loaded = True
+        screen = self.screen
+        if isinstance(screen, MainScreen):
+            screen.apply_usage()
+
     def _on_profile(self, prof: Profile) -> None:
         self.profile = prof
         screen = self.screen

nodeview_slurm-0.2.0/src/nodeview/gpu_power.py

@@ -0,0 +1,101 @@
+"""Board power of GPU models, for the energy estimate.
+
+Slurm doesn't expose it: `CurrentWatts` stays 0 without an energy plugin, and
+gres.conf has no field for it. So the model name Slurm gives (a node feature
+like 'gpu_L40S_45G', a GRES type like 'a100', 'nvidia_h100_80gb_hbm3') is
+normalized and looked up here.
+
+Values are the manufacturers' board power (TDP / TBP) in watts, per GPU as
+Slurm counts them: dual-GPU boards (K80, M60) are halved, and so on. Models
+sold in several form factors take the datacenter one (A100 is SXM, 400 W);
+the PCIe variants have their own keys when the name says so.
+"""
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+TDP_W = {
+    # --- NVIDIA datacenter ------------------------------------------------
+    'k10': 112, 'k20': 225, 'k20x': 235, 'k40': 235, 'k80': 150,
+    'm4': 50, 'm6': 100, 'm10': 56, 'm40': 250, 'm60': 150,
+    'p4': 75, 'p6': 90, 'p40': 250, 'p100': 250, 'p100sxm2': 300,
+    # 'v100sxm2' needs its own key, or the V100S would claim it as a prefix
+    'v100': 300, 'v100pcie': 250, 'v100s': 250, 'v100sxm2': 300, 'v100sxm3': 350,
+    't4': 70, 't4g': 70,
+    'a2': 60, 'a10': 150, 'a10g': 150, 'a16': 62, 'a30': 165, 'a40': 300,
+    'a100': 400, 'a100pcie': 300, 'a800': 400, 'a800pcie': 300,
+    'h100': 700, 'h100pcie': 350, 'h100nvl': 400, 'h800': 700, 'h800pcie': 350,
+    'h200': 700, 'h200nvl': 600, 'h20': 400, 'gh200': 900,
+    'l4': 72, 'l20': 275, 'l40': 300, 'l40s': 350,
+    'b100': 700, 'b200': 1000, 'gb200': 1200, 'b300': 1100, 'gb300': 1400,
+
+    # --- NVIDIA workstation -----------------------------------------------
+    'k2000': 51, 'k4000': 80, 'k5000': 122, 'k6000': 225,
+    'm2000': 75, 'm4000': 120, 'm5000': 150, 'm6000': 250,
+    'p400': 30, 'p600': 40, 'p620': 40, 'p1000': 47, 'p2000': 75, 'p2200': 75,
+    'p4000': 105, 'p5000': 180, 'p6000': 250, 'gp100': 235, 'gv100': 250,
+    # Quadro RTX (Turing): the name is just 'RTX 6000' once 'rtx' is dropped
+    '4000': 160, '5000': 230, '6000': 295, '8000': 295,
+    'a400': 50, 'a1000': 50, 'a2000': 70, 'a4000': 140, 'a4500': 200,
+    'a5000': 230, 'a5500': 230, 'a6000': 300,
+    '2000ada': 70, '4000ada': 130, '4000sff': 70, '4500ada': 210,
+    '5000ada': 250, '5880ada': 285, '6000ada': 300,
+    'pro4000': 140, 'pro4500': 200, 'pro5000': 300, 'pro6000': 600,
+    'pro6000bmaxq': 300, 'pro6000blackwellmaxq': 300,
+    'pro6000bse': 600, 'pro6000blackwellserver': 600,
+
+    # --- NVIDIA GeForce / TITAN -------------------------------------------
+    '750ti': 60, '960': 120, '970': 145, '980': 165, '980ti': 250,
+    'titanx': 250, 'titanxp': 250, 'titanv': 250, 'titanrtx': 280,
+    '1050': 75, '1050ti': 75, '1060': 120, '1070': 150, '1070ti': 180,
+    '1080': 180, '1080ti': 250,
+    '1650': 75, '1660': 120, '1660super': 125, '1660ti': 120,
+    '2060': 160, '2060super': 175, '2070': 175, '2070super': 215,
+    '2080': 215, '2080super': 250, '2080ti': 250,
+    '3050': 130, '3060': 170, '3060ti': 200, '3070': 220, '3070ti': 290,
+    '3080': 320, '3080ti': 350, '3090': 350, '3090ti': 450,
+    '4060': 115, '4060ti': 160, '4070': 200, '4070super': 220, '4070ti': 285,
+    '4070tisuper': 285, '4080': 320, '4080super': 320, '4090': 450, '4090d': 425,
+    '5060': 145, '5060ti': 180, '5070': 250, '5070ti': 300, '5080': 360, '5090': 575,
+
+    # --- AMD --------------------------------------------------------------
+    'mi25': 300, 'mi50': 300, 'mi60': 300, 'mi100': 300, 'mi210': 300,
+    'mi250': 500, 'mi250x': 560, 'mi300a': 550, 'mi300x': 750, 'mi308x': 650,
+    'mi325x': 1000, 'mi350x': 1000, 'mi355x': 1400,
+    'provii': 250, 'prov520': 225, 'prov620': 300,
+    'prow5700': 205, 'prow6600': 100, 'prow6800': 250,
+    'prow7600': 130, 'prow7700': 190, 'prow7800': 260, 'prow7900': 295,
+    'rx6800': 250, 'rx6800xt': 300, 'rx6900xt': 300, 'rx6950xt': 335,
+    'rx7800xt': 263, 'rx7900gre': 260, 'rx7900xt': 315, 'rx7900xtx': 355,
+    'rx9070': 220, 'rx9070xt': 304,
+
+    # --- Intel ------------------------------------------------------------
+    'max1100': 300, 'max1550': 600, 'flex140': 75, 'flex170': 150,
+    'a770': 225, 'a750': 225, 'b580': 190, 'gaudi2': 600, 'gaudi3': 900,
+}
+
+# vendor and family words that don't tell models apart
+_NOISE = {'nvidia', 'tesla', 'geforce', 'quadro', 'amd', 'radeon', 'instinct',
+          'intel', 'arc', 'data', 'center', 'datacenter', 'gpu', 'generation'}
+_MEMORY = re.compile(r'\d+(gb?|gib)|hbm\d?e?|g?ddr\d+x?')
+
+
+def model_key(model: str) -> str:
+    """'RTX A5000 24G' -> 'a5000'; 'nvidia_h100_80gb_hbm3' -> 'h100'."""
+    words = re.sub(r'[_\-/.]', ' ', (model or '').lower()).split()
+    words = [w for w in words if w not in _NOISE and not _MEMORY.fullmatch(w)]
+    return re.sub(r'^(rtx|gtx)', '', ''.join(words))
+
+
+def tdp_watts(model: str) -> Optional[int]:
+    """Board power of a GPU model as Slurm names it, None if unknown.
+
+    Longest matching prefix wins: '2080ti' before '2080', 'l40s' before 'l40',
+    and 'a100sxm4' still finds 'a100'.
+    """
+    key = model_key(model)
+    if not key:
+        return None
+    matches = [k for k in TDP_W if key.startswith(k)]
+    return TDP_W[max(matches, key=len)] if matches else None

{nodeview_slurm-0.1.0 → nodeview_slurm-0.2.0}/src/nodeview/i18n.py

@@ -30,6 +30,7 @@ IT = {
     'card.fit': "DOVE ENTRO ADESSO",
     'card.cluster': "CLUSTER",
     'card.account': "IL MIO ACCOUNT",
+    'card.gpu_hours': "LE MIE ORE GPU",
     'card.totals': "IL CLUSTER IN TRE NUMERI",
     'card.nodes': "NODI",
     'card.nodes_free': "NODI · solo quelli con GPU libere",
@@ -145,6 +146,23 @@ IT = {
     'acct.accounts': "  ACCOUNT                 ",
     'acct.qos': "  QOS                     ",
 
+    # --- GPU hours -------------------------------------------------------
+    'usage.none': "nessun dato: sacct non risponde",
+    'usage.last_days': "ULTIMI {n} GIORNI",
+    'usage.per_month': "ORE GPU AL MESE",
+    'usage.weekdays': "L M M G V S D",
+    'usage.months': "gen feb mar apr mag giu lug ago set ott nov dic",
+    'usage.day': "{day} {month}",
+    'usage.avg_day': "media {h} h/giorno",
+    'usage.peak': "picco {h} h il {day}",
+    'usage.col_hours': "ore GPU",
+    'usage.col_energy': "energia stimata",
+    'usage.row_month': "questo mese",
+    'usage.row_year': "anno {year}",
+    'usage.row_window': "ultimi {n} mesi",
+    'usage.row_avg': "media al mese",
+    'usage.unpriced': "escluse {h} h su GPU senza TDP noto",
+
     # --- pending reasons -------------------------------------------------
     'reason.unknown.label': "in valutazione",
     'reason.unknown.detail': "lo scheduler la sta ancora guardando",
@@ -232,6 +250,7 @@ EN = {
     'card.fit': "WHERE I FIT RIGHT NOW",
     'card.cluster': "CLUSTER",
     'card.account': "MY ACCOUNT",
+    'card.gpu_hours': "MY GPU HOURS",
     'card.totals': "THE CLUSTER IN THREE NUMBERS",
     'card.nodes': "NODES",
     'card.nodes_free': "NODES · only those with free GPUs",
@@ -340,6 +359,22 @@ EN = {
     'acct.accounts': "  ACCOUNTS                ",
     'acct.qos': "  QOS                     ",
 
+    'usage.none': "no data: sacct is not responding",
+    'usage.last_days': "LAST {n} DAYS",
+    'usage.per_month': "GPU HOURS PER MONTH",
+    'usage.weekdays': "M T W T F S S",
+    'usage.months': "Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec",
+    'usage.day': "{month} {day}",
+    'usage.avg_day': "average {h} h/day",
+    'usage.peak': "peak {h} h on {day}",
+    'usage.col_hours': "GPU hours",
+    'usage.col_energy': "estimated energy",
+    'usage.row_month': "this month",
+    'usage.row_year': "year {year}",
+    'usage.row_window': "last {n} months",
+    'usage.row_avg': "monthly average",
+    'usage.unpriced': "{h} h on GPUs with unknown TDP left out",
+
     'reason.unknown.label': "under evaluation",
     'reason.unknown.detail': "the scheduler is still looking at it",
     'reason.QOSGrpGRES.label': "group GPU quota exhausted",

{nodeview_slurm-0.1.0 → nodeview_slurm-0.2.0}/src/nodeview/nodes.py

@@ -87,12 +87,20 @@ def _gpu_indices(gres_used: str) -> list:
     return sorted(set(out))
 
 
-def _model(features: str) -> str:
-    """'gpu_RTX_A5000_24G' -> 'RTX A5000 24G'; several features -> first gpu_*"""
+def _model(features: str, gres: str = '') -> str:
+    """'gpu_RTX_A5000_24G' -> 'RTX A5000 24G'; several features -> first gpu_*
+
+    Clusters that don't name the GPU in the features often do it in the GRES
+    type instead: 'gpu:a100:4(S:0-1)' -> 'a100'.
+    """
     for feat in (features or '').split(','):
         feat = feat.strip()
         if feat.startswith('gpu_'):
             return feat[4:].replace('_', ' ')
+    for entry in (gres or '').split(','):
+        parts = entry.split('(')[0].split(':')
+        if len(parts) == 3 and parts[0] == 'gpu' and parts[1] not in ('', '(null)', 'null'):
+            return parts[1]
     first = (features or '').split(',')[0].strip()
     return t('na') if first in ('', '(null)', 'null') else first
 
@@ -296,7 +304,7 @@ def read_nodes() -> list:
             state=d.get('State', '?'),
             reason=None if reason in ('', 'none', 'N/A') else reason,
             partitions=tuple(p for p in d.get('Partitions', '').split(',') if p),
-            gpu_model=_model(d.get('AvailableFeatures', '')),
+            gpu_model=_model(d.get('AvailableFeatures', ''), d.get('Gres', '')),
             gpus_total=_gpu_count(d.get('Gres', '')),
             gpus_used=int(alloc.get('gpu', _gpu_count(d.get('GresUsed', '')))),
             gpu_indices_used=tuple(_gpu_indices(d.get('GresUsed', ''))),

{nodeview_slurm-0.1.0 → nodeview_slurm-0.2.0}/src/nodeview/render.py

@@ -9,12 +9,17 @@ the right job, and never let a color travel alone.
 """
 from __future__ import annotations
 
+import math
+from datetime import timedelta
+
 from rich import box
+from rich.columns import Columns
 from rich.console import Group
 from rich.table import Table
 from rich.text import Text
 
 from . import analysis as an
+from . import i18n
 from . import manual as manual_data
 from .i18n import t
 from . import palette
@@ -22,6 +27,8 @@ from .source import Snapshot
 
 BUSY, FREE = "●", "○"          # single GPUs: the shape tells them apart even without color
 BLOCK, EMPTY = "█", "▁"        # continuous meters
+CELL = "▆▆"                    # one day in the heatmap: the missing top leaves a gap between weeks
+EIGHTHS = " ▁▂▃▄▅▆▇█"          # bar tops, in eighths of a row
 
 # internal key -> (icon, color role). The displayed text comes from the
 # catalog: the key must not change when the language changes.
@@ -424,6 +431,186 @@ def my_account(prof):
     return Group(*blocks)
 
 
+def hours(value: float) -> str:
+    if 0 < value < 1:
+        return "<1"
+    sep = '.' if i18n.active() == 'it' else ','
+    return f"{value:,.0f}".replace(',', sep)
+
+
+def _day_label(d) -> str:
+    months = t('usage.months').split()
+    return t('usage.day', day=d.day, month=months[d.month - 1])
+
+
+def _nice_ceiling(value: float) -> float:
+    """The round number just above `value`, for the top of the axis."""
+    if value <= 0:
+        return 1.0
+    power = 10 ** math.floor(math.log10(value))
+    for step in (1, 2, 2.5, 5, 10):
+        if step * power >= value:
+            return step * power
+    return 10 * power
+
+
+def _day_heatmap(u) -> Text:
+    """Calendar of the last days: a row per week, darker means more hours.
+
+    The scale is relative to the busiest day, which the legend names. Zero is
+    a neutral grey rather than the lightest blue: "nothing" must not look like
+    "a little". Each week's total sits on the right, so the numbers are there
+    even for those who can't tell the shades apart.
+    """
+    pal = _p()
+    by_day = dict(u.days)
+    first, last = u.days[0][0], u.days[-1][0]
+    peak_day, peak = max(u.days, key=lambda dh: dh[1])
+    monday = first - timedelta(days=first.weekday())
+    weeks = [monday + timedelta(weeks=i) for i in range((last - monday).days // 7 + 1)]
+    width = max(len(_day_label(w)) for w in weeks)
+
+    def cell(h):
+        return pal.rule if h <= 0 else pal.fill(h / peak)
+
+    out = Text(no_wrap=True, overflow="crop")
+    out.append(t('usage.last_days', n=len(u.days)), style=f"bold {pal.ink_muted}")
+    out.append(f"   {hours(u.total_days)} h\n", style=f"bold {pal.ink}")
+    out.append(" " * (width + 2), style=pal.ink_muted)
+    out.append(" ".join(f"{w:<2}" for w in t('usage.weekdays').split()) + "\n",
+               style=pal.ink_muted)
+    for week in weeks:
+        out.append(f"{_day_label(week):>{width}}  ", style=pal.ink_muted)
+        total = 0.0
+        for i in range(7):
+            d = week + timedelta(days=i)
+            if first <= d <= last:
+                total += by_day[d]
+                out.append(CELL, style=cell(by_day[d]))
+            else:
+                out.append("  ")
+            out.append(" ")
+        out.append(f"{hours(total):>6} h\n", style=pal.ink_soft)
+
+    out.append(" " * (width + 2) + "0 ", style=pal.ink_muted)
+    for color in (pal.rule,) + tuple(pal.sequential):
+        out.append(CELL, style=color)
+        out.append(" ")
+    out.append(f"{hours(peak)} h\n", style=pal.ink_muted)
+    out.append(t('usage.avg_day', h=hours(u.total_days / len(u.days))), style=pal.ink_soft)
+    if peak > 0:
+        out.append("  ·  ", style=pal.ink_muted)
+        out.append(t('usage.peak', h=hours(peak), day=_day_label(peak_day)),
+                   style=pal.ink_soft)
+    return out
+
+
+def _energy_cells(values) -> tuple:
+    """Unit and 3-character cells for the energy row under the bars.
+
+    Kilowatt-hours while they fit in three digits, megawatt-hours with one
+    decimal beyond: a column is four characters, and a fourth digit would glue
+    one month to the next.
+    """
+    sep = ',' if i18n.active() == 'it' else '.'
+    if max(values, default=0) < 999.5:
+        return 'kWh', ["·" if v <= 0 else hours(v) for v in values]
+    return 'MWh', ["·" if v <= 0 else f"{v / 1000:.1f}".replace('.', sep) for v in values]
+
+
+def _month_bars(u, height: int = 8) -> Text:
+    """Vertical bars, one per month, on a single axis starting at zero.
+
+    One series, so one hue: the most visible step of the same ramp as the
+    heatmap. The axis carries its unit and the years sit on the baseline,
+    under the month they start with, so the chart reads without a legend.
+    Right under the month names, the estimated energy of each month; then a
+    small table with the totals, hours and energy side by side.
+    """
+    pal = _p()
+    values = [h for _, h in u.months]
+    top = _nice_ceiling(max(max(values), 10))    # an axis up to '<1' says nothing
+    ticks = {height: f"{hours(top)} h", height // 2: f"{hours(top / 2)} h"}
+    unit, cells = _energy_cells(u.month_kwh)
+    gutter = max(len(unit), *(len(v) for v in ticks.values()))
+    color = pal.sequential[-1] if pal.sequential else pal.ink
+
+    out = Text(no_wrap=True, overflow="crop")
+    out.append(t('usage.per_month') + "\n", style=f"bold {pal.ink_muted}")
+    for row in range(height, 0, -1):
+        tick = ticks.get(row)
+        out.append(f"{tick:>{gutter}} ┤" if tick else " " * gutter + " │", style=pal.ink_muted)
+        for v in values:
+            level = v / top * height - (row - 1)
+            if level >= 1:
+                bar = BLOCK * 3
+            elif level > 0:
+                bar = EIGHTHS[max(1, round(level * 8))] * 3
+            else:
+                bar = "   "
+            out.append(" " + bar, style=color)
+        out.append("\n")
+
+    # baseline with the years written into it: '0 └─2025──────────2026───'
+    base = ["─"] * (4 * len(values))
+    for i, (m, _) in enumerate(u.months):
+        # under January, and under the first month unless January comes right after it
+        if m.month == 1 or (i == 0 and m.month < 12):
+            base[4 * i + 1:4 * i + 5] = f"{m.year}"
+    out.append(f"{'0':>{gutter}} └", style=pal.ink_muted)
+    for ch in base:
+        out.append(ch, style=pal.ink_muted if ch == "─" else pal.ink_soft)
+    out.append("\n")
+
+    names = t('usage.months').split()
+    out.append(" " * (gutter + 2) + "".join(f" {names[m.month - 1]:<3}" for m, _ in u.months)
+               + "\n", style=pal.ink_muted)
+    out.append(f"{unit:>{gutter}}  ", style=pal.ink_muted)
+    out.append("".join(f"{c:>4}" for c in cells) + "\n", style=pal.ink_soft)
+
+    out.append("\n")
+    out.append(_totals_table(u))
+    return out
+
+
+def _totals_table(u) -> Text:
+    """Hours and estimated energy for this month, this year and the window."""
+    pal = _p()
+    n = len(u.months)
+    rows = [
+        (t('usage.row_month'), u.months[-1][1], u.month_kwh[-1]),
+        (t('usage.row_year', year=u.taken_at.year), u.year_hours, u.year_kwh),
+        (t('usage.row_window', n=n), u.total_months, sum(u.month_kwh)),
+        (t('usage.row_avg'), u.total_months / n, sum(u.month_kwh) / n),
+    ]
+    head_h, head_e = t('usage.col_hours'), t('usage.col_energy')
+    label_w = max(len(r[0]) for r in rows)
+    hours_w = max(len(head_h), *(len(hours(r[1])) + 2 for r in rows))
+    energy_w = max(len(head_e), *(len(hours(r[2])) + 4 for r in rows))
+
+    out = Text(no_wrap=True, overflow="crop")
+    out.append(" " * label_w + f"  {head_h:>{hours_w}}  {head_e:>{energy_w}}\n",
+               style=pal.ink_muted)
+    for i, (label, h, kwh) in enumerate(rows):
+        strong = i < 2
+        out.append(f"{label:<{label_w}}  ", style=pal.ink_soft)
+        out.append(f"{hours(h) + ' h':>{hours_w}}  ", style=f"bold {pal.ink}" if strong else pal.ink_soft)
+        out.append(f"{hours(kwh) + ' kWh':>{energy_w}}", style=f"bold {pal.ink}" if strong else pal.ink_soft)
+        if i < len(rows) - 1:
+            out.append("\n")
+    if u.unpriced_hours >= 1:
+        out.append("\n" + t('usage.unpriced', h=hours(u.unpriced_hours)), style=pal.ink_muted)
+    return out
+
+
+def gpu_usage(u, loaded: bool = True):
+    """GPU hours: the last days as a calendar, the last months as bars."""
+    pal = _p()
+    if u is None:
+        return Text(t('app.loading') if not loaded else t('usage.none'), style=pal.ink_muted)
+    return Columns([_day_heatmap(u), _month_bars(u)], padding=(0, 6))
+
+
 # --------------------------------------------------------------------------
 # "cluster" screen
 # --------------------------------------------------------------------------

{nodeview_slurm-0.1.0 → nodeview_slurm-0.2.0}/src/nodeview/snapshot.py

@@ -67,6 +67,8 @@ def print_once(width=None, cluster=False) -> int:
         console.print(_panel(render.my_jobs(snap), t('card.my_jobs')))
         console.print(_panel(render.where_do_i_fit(snap), t('card.fit')))
         console.print(_panel(render.cluster(snap), t('card.cluster')))
+        from .usage import read_gpu_usage
+        console.print(_panel(render.gpu_usage(read_gpu_usage()), t('card.gpu_hours')))
         from .profile import read_profile
         console.print(_panel(render.my_account(read_profile()), t('card.account')))
     return 0

nodeview_slurm-0.2.0/src/nodeview/usage.py

@@ -0,0 +1,204 @@
+"""GPU hours you've used, per day and per month, and the energy they took.
+
+A single `sacct` call covers the whole window (a year of jobs and steps takes
+~0.6 s) and each job's GPUs × time is split across the days and months it
+overlaps. That's what `sreport` does too, but sreport rolls up once an hour and
+misses the jobs still running: here today is already counted.
+
+`-D` matters: without it a requeued job shows only its last run, and the hours
+of the earlier ones vanish.
+
+Energy is an estimate, not a measurement: without an energy plugin Slurm
+records `ConsumedEnergy=0`. It combines what Slurm does know, the GPU model of
+the node and each job's average GPU utilization, with the model's board power.
+"""
+from __future__ import annotations
+
+import re
+import subprocess
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta
+from typing import Optional
+
+from .gpu_power import tdp_watts
+from .nodes import _expand_nodelist, read_nodes, short_name
+
+DAYS = 30
+MONTHS = 12
+
+_GPUS = re.compile(r'(?:^|,)gres/gpu=(\d+)')
+_UTIL = re.compile(r'(?:^|,)gres/gpuutil=(\d+)')
+_TYPED = re.compile(r'(?:^|,)gres/gpu:([^=,]+)=\d+')     # 'gres/gpu:a100=2'
+
+# An allocated GPU draws power even at 0% utilization: roughly this share of
+# its board power, the rest grows with utilization.
+IDLE_SHARE = 0.15
+
+
+@dataclass(frozen=True)
+class GpuUsage:
+    days: tuple          # ((date, hours), ...) oldest first, ending today
+    months: tuple        # ((first day of the month, hours), ...) oldest first
+    month_kwh: tuple     # estimated energy, aligned with `months`
+    month_priced: tuple  # the GPU hours behind `month_kwh`: those with a known model
+    taken_at: datetime
+
+    @property
+    def total_days(self) -> float:
+        return sum(h for _, h in self.days)
+
+    @property
+    def total_months(self) -> float:
+        return sum(h for _, h in self.months)
+
+    def _this_year(self, values) -> float:
+        return sum(v for (m, _), v in zip(self.months, values) if m.year == self.taken_at.year)
+
+    @property
+    def year_kwh(self) -> float:
+        """January to December of the current year: the window always covers it."""
+        return self._this_year(self.month_kwh)
+
+    @property
+    def year_hours(self) -> float:
+        return self._this_year(h for _, h in self.months)
+
+    @property
+    def unpriced_hours(self) -> float:
+        """GPU hours on models without a known board power, left out of the energy."""
+        return self._this_year(h for _, h in self.months) - self._this_year(self.month_priced)
+
+
+def _month_start(d: date, back: int = 0) -> date:
+    index = d.year * 12 + (d.month - 1) - back
+    return date(index // 12, index % 12 + 1, 1)
+
+
+def _ts(value: str) -> Optional[datetime]:
+    try:
+        return datetime.strptime(value.strip(), '%Y-%m-%dT%H:%M:%S')
+    except ValueError:
+        return None              # 'None' / 'Unknown': not started yet
+
+
+def _gpu_count(tres: str) -> int:
+    """GPUs in an AllocTRES: the plain 'gres/gpu=N', or the typed ones summed
+    when the plain one is missing."""
+    match = _GPUS.search(tres)
+    if match:
+        return int(match.group(1))
+    return sum(int(m.group(1)) for m in re.finditer(r'(?:^|,)gres/gpu:[^=,]+=(\d+)', tres))
+
+
+def _runs(text: str, now: datetime, node_tdp: dict) -> list:
+    """(start, end, gpus, watts) for every run that held at least one GPU.
+
+    Allocation lines give time, GPUs and node; step lines ('123.batch', '123.0')
+    carry the measured utilization, and the busiest step stands for the job.
+    The GPU model comes from the node, or failing that from a typed GRES in
+    the job ('gres/gpu:a100=2'); `watts` is None when neither is known. Jobs without a
+    utilization reading (too short, still running) get the average of the
+    others, weighted by GPU hours.
+    """
+    allocs, util = [], {}
+    for line in text.splitlines():
+        parts = line.split('|')
+        if len(parts) < 6:
+            continue
+        jobid, start, end, tres, nodelist, usage = parts[:6]
+        if '.' in jobid:
+            match = _UTIL.search(usage)
+            if match:
+                base = jobid.split('.')[0]
+                util[base] = max(util.get(base, 0), int(match.group(1)))
+            continue
+        begin = _ts(start)
+        if begin is None:
+            continue
+        gpus = _gpu_count(tres)
+        finish = _ts(end) or now          # still running
+        if gpus and finish > begin:
+            nodes = _expand_nodelist(nodelist)
+            tdp = node_tdp.get(short_name(nodes[0])) if nodes else None
+            typed = _TYPED.search(tres)
+            if tdp is None and typed:
+                tdp = tdp_watts(typed.group(1))
+            allocs.append((jobid, begin, finish, gpus, tdp))
+
+    def share(jobid, gpus):
+        value = util[jobid]
+        return min(100, value / gpus if value > 100 else value) / 100
+
+    known = [(a, (a[2] - a[1]).total_seconds() * a[3]) for a in allocs if a[0] in util]
+    weight = sum(w for _, w in known)
+    fallback = sum(share(a[0], a[3]) * w for a, w in known) / weight if weight else 0.5
+
+    out = []
+    for jobid, begin, finish, gpus, tdp in allocs:
+        u = share(jobid, gpus) if jobid in util else fallback
+        watts = gpus * tdp * (IDLE_SHARE + (1 - IDLE_SHARE) * u) if tdp else None
+        out.append((begin, finish, gpus, watts))
+    return out
+
+
+def _spread(intervals: list, edges: list) -> list:
+    """Sum of weight × hours falling in each [edges[i], edges[i+1]) bucket.
+
+    `intervals` holds (start, end, weight): GPUs for GPU hours, kW for kWh.
+    """
+    totals = [0.0] * (len(edges) - 1)
+    lo, hi = edges[0], edges[-1]
+    for start, end, weight in intervals:
+        start, end = max(start, lo), min(end, hi)
+        if end <= start:
+            continue
+        for i in range(len(totals)):
+            a, b = max(start, edges[i]), min(end, edges[i + 1])
+            if b > a:
+                totals[i] += weight * (b - a).total_seconds() / 3600
+    return totals
+
+
+def summarize(text: str, now: datetime, node_tdp: Optional[dict] = None) -> GpuUsage:
+    today = now.date()
+    day_starts = [today - timedelta(days=DAYS - 1 - i) for i in range(DAYS)]
+    month_starts = [_month_start(today, MONTHS - 1 - i) for i in range(MONTHS)]
+    runs = _runs(text, now, node_tdp or {})
+
+    def edges(starts, after):
+        return [datetime.combine(d, datetime.min.time()) for d in starts + [after]]
+
+    by_day = edges(day_starts, today + timedelta(days=1))
+    by_month = edges(month_starts, _month_start(today, -1))
+    gpus = [(s, e, g) for s, e, g, _ in runs]
+    priced = [(s, e, g) for s, e, g, w in runs if w is not None]
+    kilowatts = [(s, e, w / 1000) for s, e, _, w in runs if w is not None]
+    return GpuUsage(days=tuple(zip(day_starts, _spread(gpus, by_day))),
+                    months=tuple(zip(month_starts, _spread(gpus, by_month))),
+                    month_kwh=tuple(_spread(kilowatts, by_month)),
+                    month_priced=tuple(_spread(priced, by_month)),
+                    taken_at=now)
+
+
+def _node_tdp() -> dict:
+    """Node name -> board power of its GPUs; empty if the nodes can't be read."""
+    try:
+        return {n.name: tdp_watts(n.gpu_model) for n in read_nodes()}
+    except Exception:
+        return {}
+
+
+def read_gpu_usage() -> Optional[GpuUsage]:
+    now = datetime.now()
+    since = min(_month_start(now.date(), MONTHS - 1),
+                now.date() - timedelta(days=DAYS - 1))
+    try:
+        out = subprocess.run(
+            ['sacct', '-n', '-D', '-P', '-S', since.isoformat(), '-E', 'now',
+             '-o', 'JobID,Start,End,AllocTRES,NodeList,TRESUsageInAve'],
+            capture_output=True, text=True, timeout=25)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if out.returncode != 0:
+        return None
+    return summarize(out.stdout, now, _node_tdp())