scopos 2.0.0__py3-none-any.whl

scopos/__init__.py

@@ -0,0 +1,10 @@
+# src/scopos/__init__.py
+"""
+SCOPOS
+======
+
+a Textual TUI for monitoring GPU memory usage per user.
+"""
+
+__author__ = "Zhen Tian"
+__version__ = "2.0.0"

scopos/app.py

@@ -0,0 +1,185 @@
+# -*- coding: utf-8 -*-
+"""The Scopos Textual application."""
+
+from __future__ import annotations
+import time
+from rich.text import Text
+from textual.app import (App, ComposeResult)
+from textual.containers import (Container, Horizontal, VerticalScroll)
+from textual.widgets import (Footer, Static)
+from typing import (Dict, List)
+
+from . import __version__
+from .monitor import GPUInfo, Monitor
+from .widgets import GpuCard, Logo, SysMeter
+
+
+class Clock(Static):
+    """Date / time / version, pinned top-right."""
+
+    def on_mount(self):
+        self.update_clock()
+        self.set_interval(1.0, self.update_clock)
+
+    def update_clock(self):
+        now = time.localtime()
+        text = Text(justify="left")
+        text.append(time.strftime("%Y-%m-%d  ", now), style="bold")
+        text.append(time.strftime("%H:%M:%S", now), style="bold cyan")
+        self.update(text)
+
+
+class ScoposApp(App):
+    """Monitor GPU memory usage, grouped by user."""
+
+    TITLE = "SCOPOS"
+
+    # Roughly the narrowest a card stays readable; used to pick column count.
+    # The full COMMAND column needs room, so cards stay wide and only tile into
+    # multiple columns on genuinely wide terminals.
+    CARD_MIN_WIDTH = 100
+
+    CSS = """
+    Screen {
+        layout: vertical;
+    }
+    #topbar {
+        height: 5;
+        padding: 0 1;
+        background: $panel;
+    }
+    #topbar Logo {
+        width: auto;
+        height: 5;
+        content-align: left top;
+    }
+    #topbar Clock {
+        width: auto;
+        height: 4;
+        padding-bottom: 0;
+        content-align: center bottom;
+    }
+    #topbar #spacer1 {
+        width: 1fr;
+    }
+    #topbar #spacer2 {
+            width: 1fr;
+        }
+    #topbar SysMeter {
+        width: auto;
+        height: 5;
+        padding-right: 4;
+        padding-bottom: 1;
+        content-align: right bottom;
+    }
+    #grid {
+        layout: grid;
+        grid-size: 1;
+        grid-rows: auto;
+        grid-gutter: 1 2;
+        height: auto;
+        padding: 1 2;
+    }
+    #body {
+        height: 1fr;
+    }
+    #status {
+        height: 1;
+        padding: 0 2;
+        color: $text-muted;
+    }
+    """
+
+    BINDINGS = [
+        ("q", "quit", "Quit"),
+        ("r", "refresh", "Refresh now"),
+        ("d", "toggle_dark", "Light/Dark"),
+    ]
+
+    def __init__(self, watch_user: str = "", interval: int = 5, demo: bool = False):
+        super().__init__()
+        self.interval = max(1, interval)
+        self.monitor = Monitor(watch_user=watch_user, demo=demo)
+        self.show_detail = bool(self.monitor.watch_user)
+        self._cards: Dict[int, GpuCard] = {}
+
+    def compose(self) -> ComposeResult:
+        with Horizontal(id="topbar"):
+            yield Logo()
+            yield Static(id="spacer1")
+            yield Clock()
+            yield Static(id="spacer2")
+            yield SysMeter(self.monitor)
+        with VerticalScroll(id="body"):
+            yield Container(id="grid")
+        yield Static(id="status")
+        yield Footer()
+
+    def on_mount(self):
+        self.refresh_data()
+        self.set_interval(self.interval, self.refresh_data)
+
+    def on_resize(self):
+        self._relayout_columns()
+
+    # -- layout ------------------------------------------------------------
+    def _relayout_columns(self):
+        if not self._cards:
+            return
+        width = self.size.width
+        cols = max(1, width // self.CARD_MIN_WIDTH)
+        cols = min(cols, len(self._cards))
+        grid = self.query_one("#grid")
+        grid.styles.grid_size_columns = cols
+
+    # -- data --------------------------------------------------------------
+    def action_refresh(self):
+        self.refresh_data()
+
+    def refresh_data(self):
+        try:
+            gpus = self.monitor.collect()
+        except Exception as exc:  # keep the UI alive on transient NVML errors
+            self.query_one("#status", Static).update(
+                Text(f"collection error: {exc}", style="red")
+            )
+            return
+        self._sync_cards(gpus)
+        for gpu in gpus:
+            self._cards[gpu.index].update(gpu)
+        self._update_status(gpus)
+
+    def _sync_cards(self, gpus: List[GPUInfo]):
+        wanted = {g.index for g in gpus}
+        if wanted == set(self._cards):
+            return
+        # GPU set changed (first run, or hot-plug): rebuild the grid.
+        grid = self.query_one("#grid")
+        grid.remove_children()
+        self._cards.clear()
+        for gpu in gpus:
+            card = GpuCard(self.monitor, self.show_detail)
+            self._cards[gpu.index] = card
+            grid.mount(card)
+        self.call_after_refresh(self._relayout_columns)
+
+    def _update_status(self, gpus: List[GPUInfo]):
+        n_proc = sum(len(g.procs) for g in gpus)
+        users = {p.user for g in gpus for p in g.procs}
+        mode = "demo" if self.monitor.demo else "live"
+        watch = (
+            f"  ·  watching [{self.monitor.watch_user}]"
+            if self.monitor.watch_user
+            else ""
+        )
+        text = Text()
+        text.append(f"{len(gpus)} GPU · {n_proc} proc · {len(users)} users")
+        text.append(
+            f"  ·  refresh {self.interval}s  ·  {mode}{watch}"
+            "  ·  click a column header to sort",
+            style="dim",
+        )
+        self.query_one("#status", Static).update(text)
+
+    def on_unmount(self):
+        self.monitor.stop()

scopos/cli.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+"""Command line entry point: ``python -m scopos``."""
+
+from __future__ import annotations
+import argparse
+
+from . import __version__
+from .app import ScoposApp
+
+
+def parse_args(argv=None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="scopos",
+        description="Monitor GPU memory usage, grouped by user (Textual TUI).",
+    )
+    parser.add_argument(
+        "-u",
+        "--user",
+        default="",
+        help="Highlight this user and show their shell-script task details.",
+    )
+    parser.add_argument(
+        "-i",
+        "--interval",
+        type=int,
+        default=5,
+        help="Refresh interval in seconds (default: 5).",
+    )
+    parser.add_argument(
+        "--demo",
+        action="store_true",
+        help="Run with synthetic GPU data (no NVIDIA driver required).",
+    )
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version=f"scopos {__version__}",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    args = parse_args(argv)
+    app = ScoposApp(
+        watch_user=args.user,
+        interval=args.interval,
+        demo=args.demo,
+    )
+    app.run()

scopos/monitor.py

@@ -0,0 +1,393 @@
+# -*- coding: utf-8 -*-
+"""Data collection layer for Scopos.
+
+This module is intentionally free of any UI code so it can be reused or
+tested on its own. :class:`Monitor` keeps a small amount of state between
+refreshes so that a given user always keeps the same colour and the same
+process numbering, exactly like the original CLI did.
+"""
+
+from __future__ import annotations
+import re
+import time
+import random
+from dataclasses import (dataclass, field)
+from typing import (Dict, List, Optional)
+
+try:  # pynvml is only available on machines with an NVIDIA driver.
+    import pynvml as pn
+except Exception:  # pragma: no cover - exercised only without a driver.
+    pn = None
+
+try:
+    import psutil
+except Exception:  # pragma: no cover
+    psutil = None
+
+
+# A palette of visually distinct colours assigned to users in order of
+# first appearance. Names are Rich/Textual colour names so they render the
+# same in tables, bars and legends.
+USER_PALETTE: List[str] = [
+    # "bright_red",
+    "bright_green",
+    "bright_yellow",
+    "bright_blue",
+    "bright_magenta",
+    "bright_cyan",
+    "orange1",
+    "spring_green2",
+    "deep_pink2",
+    "gold1",
+    "dodger_blue1",
+    "medium_purple1",
+    "chartreuse2",
+    "hot_pink",
+]
+
+
+@dataclass
+class ProcInfo:
+    """A single compute process running on a GPU."""
+
+    pid: int
+    name: str
+    user: str
+    mem: int  # bytes of GPU memory used
+    started: str  # parent-process creation time, formatted
+    runtime: str  # how long this process has been running, formatted
+    number: str  # per-user "parentNo-childNo" label, filled by Monitor
+    detail: str  # script/task detail (only filled for the watched user)
+    cmd: str = ""  # full command line, including arguments
+    ppid: int = 0  # parent pid, used for per-user numbering
+    started_ts: float = 0.0  # raw parent start time, for sorting
+    runtime_sec: int = 0  # raw runtime in seconds, for sorting
+
+
+@dataclass
+class GPUInfo:
+    """A snapshot of one GPU and the processes running on it."""
+
+    index: int
+    name: str
+    mem_used: int
+    mem_total: int
+    mem_free: int
+    util: int  # core utilisation %, -1 if unknown
+    temperature: int  # degrees C, -1 if unknown
+    procs: List[ProcInfo] = field(default_factory=list)
+    user_mems: Dict[str, int] = field(default_factory=dict)
+
+    @property
+    def idle_rate(self) -> float:
+        return self.mem_free / self.mem_total if self.mem_total else 0.0
+
+    @property
+    def used_rate(self) -> float:
+        return self.mem_used / self.mem_total if self.mem_total else 0.0
+
+    def mvp(self) -> Optional[str]:
+        """Return the user holding the most memory on this GPU, if any."""
+        if not self.user_mems:
+            return None
+        return max(self.user_mems.items(), key=lambda kv: kv[1])[0]
+
+
+def fmt_gb(num_bytes: float) -> str:
+    return "%.2f" % (num_bytes / (1024 ** 3))
+
+
+def fmt_duration(seconds: int) -> str:
+    """Format a time span with unit symbols, e.g. "2d 03h", "3h 20m", "45s"."""
+    seconds = max(0, int(seconds))
+    d, rem = divmod(seconds, 86400)
+    h, rem = divmod(rem, 3600)
+    m, s = divmod(rem, 60)
+    if d:
+        return f"{d}d {h:02d}h"
+    if h:
+        return f"{h}h {m:02d}m"
+    if m:
+        return f"{m}m {s:02d}s"
+    return f"{s}s"
+
+
+class Monitor:
+    """Collects GPU snapshots, keeping per-user state stable across refreshes."""
+
+    def __init__(self, watch_user: str = "", demo: bool = False):
+        self.watch_user = watch_user.strip()
+        self.demo = demo
+        # username -> colour, assigned on first sight and kept forever.
+        self._user_colors: Dict[str, str] = {}
+        self._next_color = 0
+        self._initialised = False
+        if self.watch_user:
+            # Make sure the watched user always gets `bright_red`.
+            self._user_colors.setdefault(self.watch_user, "bright_red")
+
+    # -- colours -----------------------------------------------------------
+    def color_for(self, user: str) -> str:
+        if user not in self._user_colors:
+            color = USER_PALETTE[self._next_color % len(USER_PALETTE)]
+            self._user_colors[user] = color
+            self._next_color += 1
+        return self._user_colors[user]
+
+    # -- lifecycle ---------------------------------------------------------
+    def start(self):
+        if not self.demo and not self._initialised and pn is not None:
+            pn.nvmlInit()
+            self._initialised = True
+
+    def stop(self):
+        if not self.demo and self._initialised and pn is not None:
+            try:
+                pn.nvmlShutdown()
+            except Exception:
+                pass
+            self._initialised = False
+
+    # -- system memory -----------------------------------------------------
+    def system_stats(self) -> Dict[str, tuple]:
+        """Return host RAM/swap usage as {"mem": (used, total), "swap": (...)}.
+
+        Falls back to plausible synthetic values when psutil is unavailable
+        (e.g. demo mode on a machine without it installed).
+        """
+        if psutil is not None:
+            vm = psutil.virtual_memory()
+            sm = psutil.swap_memory()
+            return {"mem": (vm.used, vm.total), "swap": (sm.used, sm.total)}
+        rng = random.Random()
+        mem_total = 32 * 1024 ** 3
+        swap_total = 8 * 1024 ** 3
+        return {
+            "mem": (int(mem_total * rng.uniform(0.2, 0.8)), mem_total),
+            "swap": (int(swap_total * rng.uniform(0.0, 0.4)), swap_total),
+        }
+
+    # -- collection --------------------------------------------------------
+    def collect(self) -> List[GPUInfo]:
+        if self.demo:
+            return self._collect_demo()
+        return self._collect_real()
+
+    # -- real collection ---------------------------------------------------
+    def _collect_real(self) -> List[GPUInfo]:
+        self.start()
+        gpus: List[GPUInfo] = []
+
+        for gpu_id in range(pn.nvmlDeviceGetCount()):
+            handle = pn.nvmlDeviceGetHandleByIndex(gpu_id)
+            name = _decode(pn.nvmlDeviceGetName(handle))
+            mem = pn.nvmlDeviceGetMemoryInfo(handle)
+            used, free = int(mem.used), int(mem.free)
+            total = used + free
+            util, temp = -1, -1
+            try:
+                util = pn.nvmlDeviceGetUtilizationRates(handle).gpu
+            except Exception:
+                pass
+            try:
+                temp = pn.nvmlDeviceGetTemperature(handle, pn.NVML_TEMPERATURE_GPU)
+            except Exception:
+                pass
+
+            gpu = GPUInfo(gpu_id, name, used, total, free, util, temp)
+
+            try:
+                processes = pn.nvmlDeviceGetComputeRunningProcesses_v2(handle)
+            except Exception:
+                processes = []
+
+            for process in processes:
+                info = self._build_proc(process)
+                if info is None:
+                    continue
+                gpu.procs.append(info)
+                gpu.user_mems[info.user] = (
+                    gpu.user_mems.get(info.user, 0) + info.mem
+                )
+            gpus.append(gpu)
+        self._assign_numbers(gpus)
+        return gpus
+
+    def _assign_numbers(self, gpus: List[GPUInfo]) -> None:
+        """Fill in each process's "parentNo-childNo" label.
+
+        Numbering is per user and spans every GPU: a user's parent processes
+        are numbered in the order they are first seen across all GPUs, and the
+        child counter increments for every process sharing that parent.
+        """
+        user_ppids: Dict[str, List[int]] = {}
+        child_count: Dict[tuple, int] = {}
+        for gpu in gpus:
+            for proc in gpu.procs:
+                ppids = user_ppids.setdefault(proc.user, [])
+                if proc.ppid not in ppids:
+                    ppids.append(proc.ppid)
+                pp_no = ppids.index(proc.ppid) + 1
+                key = (proc.user, proc.ppid)
+                child_count[key] = child_count.get(key, 0) + 1
+                proc.number = f"{pp_no:02d}-{child_count[key]:02d}"
+
+    def _build_proc(self, process) -> Optional[ProcInfo]:
+        try:
+            pid = int(process.pid)
+            p = psutil.Process(pid)
+        except Exception:
+            return None
+        started_ts = 0.0
+        try:
+            ppid = p.ppid()
+            pp = psutil.Process(ppid)
+            started_ts = pp.create_time()
+            started = time.strftime("%y-%m-%d %H:%M:%S", time.localtime(started_ts))
+        except Exception:
+            ppid = 0
+            pp = None
+            started = "?"
+
+        runtime_sec = int(time.time() - p.create_time())
+        runtime = fmt_duration(runtime_sec)
+        try:
+            user = p.username()
+        except Exception:
+            user = "?"
+        self.color_for(user)
+
+        try:
+            cmd = " ".join(p.cmdline()).strip()
+        except Exception:
+            cmd = ""
+        if not cmd:
+            cmd = p.name()
+
+        mem = int(process.usedGpuMemory or 0)
+
+        detail = "-"
+        if user == self.watch_user and pp is not None:
+            detail = self._script_detail(p, pp)
+
+        # number is assigned later, once every GPU has been collected.
+        return ProcInfo(
+            pid, p.name(), user, mem, started, runtime, "", detail,
+            cmd=cmd, ppid=ppid, started_ts=started_ts, runtime_sec=runtime_sec,
+        )
+
+    def _script_detail(self, p, pp) -> str:
+        """Best-effort reconstruction of which task in a shell script is running.
+
+        Ported from the original tool; wrapped so any failure simply shows "?".
+        """
+        try:
+            pp_file_path = pp.open_files()[0].path
+            pp_file_name = pp_file_path.rsplit("/", maxsplit=1)[-1]
+            cur_cmd = " ".join(p.cmdline())
+            total_task = 0
+            cur_task = -1
+            bash_args: Dict[str, str] = {}
+
+            def replace_bash_args(cmd: str) -> str:
+                for arg, val in bash_args.items():
+                    rx = re.compile(r"\$(\{" + arg + r"\}|" + arg + r"(?!_))")
+                    cmd = rx.sub(val, cmd)
+                return cmd.replace('"', "")
+
+            with open(pp_file_path, "r", newline=None) as fh:
+                for cmd in fh:
+                    if cmd.startswith("#"):
+                        continue
+                    cmd = cmd.strip("\n")
+                    if cmd.startswith(p.name()):
+                        total_task += 1
+                        if replace_bash_args(cmd) == cur_cmd:
+                            cur_task = total_task
+                    elif "=" in cmd:
+                        key, raw = cmd.split("=", maxsplit=1)
+                        if "$" in raw:
+                            val = replace_bash_args(raw)
+                            if "$" in val:
+                                raise NotImplementedError
+                        else:
+                            val = raw
+                        bash_args[key] = val.strip('"')
+            return f"{pp_file_name} [{cur_task}/{total_task}]"
+        except Exception:
+            return "?"
+
+    # -- demo collection ---------------------------------------------------
+    def _collect_demo(self) -> List[GPUInfo]:
+        rng = random.Random()  # fresh randomness each tick for a "live" feel
+        names = [
+            "NVIDIA GeForce RTX 4090",
+            "NVIDIA A100-SXM4-80GB",
+            "NVIDIA H100 80GB HBM3",
+        ]
+        users_pool = ["alice", "bob", "carol", "dave", "erin", self.watch_user or "frank"]
+        # Give each user a small pool of "parent" pids so the same parent can
+        # show up on several GPUs - that is what makes the per-user numbering
+        # (parentNo-childNo) interesting to look at.
+        parent_pids = {u: [rng.randint(1000, 9999) for _ in range(2)] for u in users_pool}
+        gpus: List[GPUInfo] = []
+        n_gpu = 4
+        for gpu_id in range(n_gpu):
+            total = rng.choice([24, 40, 80]) * 1024 ** 3
+            gpu = GPUInfo(
+                gpu_id,
+                names[gpu_id % len(names)],
+                0,
+                total,
+                total,
+                rng.randint(0, 100),
+                rng.randint(35, 85),
+            )
+            used = 0
+            n_proc = rng.randint(0, 5)
+            for _ in range(n_proc):
+                user = rng.choice(users_pool)
+                self.color_for(user)
+                mem = rng.randint(1, 12) * 1024 ** 3
+                if used + mem > total:
+                    break
+                used += mem
+                detail = "-"
+                if user == self.watch_user:
+                    detail = f"train_{rng.randint(1,9)}.sh [{rng.randint(1,4)}/4]"
+                runtime_sec = rng.randint(0, 400000)
+                started_ts = time.time() - runtime_sec
+                script = rng.choice(["train.py", "finetune.py", "eval.py", "main.py"])
+                cmd = (
+                    f"python {script} --lr {rng.choice(['1e-3', '5e-4', '3e-5'])}"
+                    f" --batch-size {rng.choice([16, 32, 64])}"
+                    f" --epochs {rng.randint(10, 200)} --fp16"
+                )
+                gpu.procs.append(
+                    ProcInfo(
+                        pid=rng.randint(10000, 99999),
+                        name=rng.choice(["python", "python3", "train", "pt_main"]),
+                        user=user,
+                        mem=mem,
+                        started=time.strftime("%y-%m-%d %H:%M:%S", time.localtime(started_ts)),
+                        runtime=fmt_duration(runtime_sec),
+                        number="",
+                        detail=detail,
+                        cmd=cmd,
+                        ppid=rng.choice(parent_pids[user]),
+                        started_ts=started_ts,
+                        runtime_sec=runtime_sec,
+                    )
+                )
+                gpu.user_mems[user] = gpu.user_mems.get(user, 0) + mem
+            gpu.mem_used = used
+            gpu.mem_free = total - used
+            gpus.append(gpu)
+        self._assign_numbers(gpus)
+        return gpus
+
+
+def _decode(value) -> str:
+    if isinstance(value, bytes):
+        return value.decode("utf-8", "replace")
+    return str(value)

scopos/widgets.py

@@ -0,0 +1,308 @@
+# -*- coding: utf-8 -*-
+"""Reusable Textual widgets for Scopos."""
+
+from __future__ import annotations
+from rich.text import Text
+from textual.containers import Vertical
+from textual.widget import Widget
+from textual.widgets import (DataTable, Static)
+from typing import (Callable, List, Optional, Tuple)
+
+from . import __version__
+from .monitor import (GPUInfo, Monitor, fmt_gb)
+
+
+LOGO = r"""  ___   ___  _____  ____  _____  ___
+ / __) / __)(  _  )(  _ \(  _  )/ __)
+ \__ \( (__  )(_)(  )___/ )(_)( \__ \
+ (___/ \___)(_____)(__)  (_____)(___/"""
+
+
+class Logo(Static):
+    """The SCOPOS ASCII logo, pinned top-left."""
+
+    def __init__(self):
+        text = Text(LOGO, style="bold cyan")
+        text.append(f"  v{__version__}", style="dim white")
+        super().__init__(text)
+
+
+class SysMeter(Static):
+    """Compact host RAM / swap usage bars, shown next to the logo."""
+
+    DEFAULT_CSS = """
+    SysMeter {
+        width: auto;
+        height: auto;
+    }
+    """
+
+    BAR_WIDTH = 26
+
+    def __init__(self, monitor: Monitor) -> None:
+        super().__init__()
+        self.monitor = monitor
+
+    def on_mount(self) -> None:
+        self.refresh_stats()
+        self.set_interval(2.0, self.refresh_stats)
+
+    def refresh_stats(self) -> None:
+        stats = self.monitor.system_stats()
+        text = Text(justify="right")
+        text.append(self._line("Mem", *stats["mem"]))
+        text.append("\n")
+        text.append(self._line("Swp", *stats["swap"]))
+        self.update(text)
+
+    def _line(self, label: str, used: float, total: float) -> Text:
+        total = total or 1
+        frac = max(0.0, min(1.0, used / total))
+        if frac >= 0.85:
+            color = "red"
+        elif frac >= 0.6:
+            color = "yellow"
+        else:
+            color = "green"
+        filled = round(frac * self.BAR_WIDTH)
+        gb = 1024 ** 3
+        line = Text()
+        line.append(f"{label} ", style="bold")
+        line.append("▕", style="grey50")
+        line.append("█" * filled, style=color)
+        line.append("░" * (self.BAR_WIDTH - filled), style="grey35")
+        line.append("▏", style="grey50")
+        line.append(f" {used / gb:5.1f} / {total / gb:5.1f} GB", style="dim")
+        line.append(f" {frac * 100:3.0f}%", style=color)
+        return line
+
+
+class MemoryBar(Widget):
+    """A single-line bar whose coloured segments show each user's share.
+
+    The bar always fills its own width, so it grows and shrinks with the
+    terminal - that is what gives the "直观占比" (intuitive proportion) view.
+    """
+
+    DEFAULT_CSS = """
+    MemoryBar {
+        height: 1;
+        width: 1fr;
+    }
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._segments: List[Tuple[str, float]] = []
+        self._total: float = 1.0
+
+    def set_data(self, segments: List[Tuple[str, float]], total: float):
+        """segments: list of (colour, weight); total: the bar's full weight."""
+        self._segments = segments
+        self._total = total or 1.0
+        self.refresh()
+
+    def render(self) -> Text:
+        width = self.size.width or 1
+        text = Text(no_wrap=True, overflow="crop")
+        used = 0
+        for color, weight in self._segments:
+            if weight <= 0:
+                continue
+            cells = round(weight / self._total * width)
+            if cells == 0:
+                cells = 1  # keep tiny-but-present users visible
+            cells = min(cells, width - used)
+            if cells <= 0:
+                break
+            text.append("█" * cells, style=color)
+            used += cells
+        if used < width:
+            text.append("░" * (width - used), style="grey35")
+        return text
+
+
+class GpuCard(Vertical):
+    """One GPU: header, stats line, proportion bar, legend and process table."""
+
+    DEFAULT_CSS = """
+    GpuCard {
+        height: auto;
+        border: round $primary;
+        border-title-color: $text;
+        border-title-style: bold;
+        padding: 0 1;
+        margin: 0;
+    }
+    GpuCard .stats { height: 1; }
+    GpuCard .legend { height: auto; color: $text-muted; }
+    GpuCard DataTable {
+        height: auto;
+        max-height: 30;
+        margin-top: 1;
+    }
+    """
+
+    # Header labels and, for each, how to sort the rows by that column.
+    # ``None`` means the column is not sortable. The trailing DETAIL column is
+    # only shown when a user is being watched (see ``_headers``).
+    COLUMNS: List[Tuple[str, Optional[Callable]]] = [
+        ("PID", lambda p: p.pid),
+        ("USER", lambda p: p.user.lower()),
+        ("NO.", lambda p: (p.user.lower(), p.number)),
+        ("MEM/GB", lambda p: p.mem),
+        ("STARTED", lambda p: p.started_ts),
+        ("RUNTIME", lambda p: p.runtime_sec),
+        ("DETAIL", lambda p: (p.detail or "").lower()),
+        ("COMMAND", lambda p: p.cmd.lower()),
+    ]
+    # Columns that read most naturally largest-first on the initial click:
+    # PID, MEM/GB, STARTED, RUNTIME.
+    DESC_FIRST = {0, 3, 4, 5}
+
+    def __init__(self, monitor: Monitor, show_detail: bool) -> None:
+        super().__init__()
+        self.monitor = monitor
+        self.stats = Static(classes="stats")
+        self.bar = MemoryBar()
+        self.legend = Static(classes="legend")
+        self.table = DataTable(zebra_stripes=True, cursor_type="row")
+        self._pending: Optional[GPUInfo] = None
+        self._gpu: Optional[GPUInfo] = None
+        self._sort_index: Optional[int] = None
+        self._sort_reverse: bool = False
+        # detail
+        self.show_detail = show_detail
+        if not show_detail:
+            self._header = self.COLUMNS[:-2] + self.COLUMNS[-1:]
+        else:
+            self._header = self.COLUMNS
+
+    def compose(self):
+        yield self.stats
+        yield self.bar
+        yield self.legend
+        yield self.table
+
+    def on_mount(self) -> None:
+        if self._pending is not None:
+            self._apply(self._pending)
+
+    # -- sorting -----------------------------------------------------------
+    def on_data_table_header_selected(
+        self, event: DataTable.HeaderSelected
+    ) -> None:
+        event.stop()
+        idx = event.column_index
+        if idx >= len(self._header) or self._header[idx][1] is None:
+            return
+        if self._sort_index == idx:
+            self._sort_reverse = not self._sort_reverse
+        else:
+            self._sort_index = idx
+            self._sort_reverse = idx in self.DESC_FIRST
+        if self._gpu is not None:
+            self._update_table(self._gpu)
+
+    # -- updating ----------------------------------------------------------
+    def update(self, gpu: GPUInfo):
+        # A card may be updated in the same frame it is mounted, before its
+        # columns exist; defer until on_mount in that case.
+        if not self.is_mounted:
+            self._pending = gpu
+            return
+        self._apply(gpu)
+
+    def _apply(self, gpu: GPUInfo):
+        self._pending = None
+        self._gpu = gpu
+        self.border_title = f" #{gpu.index}  {gpu.name} "
+        self._update_stats(gpu)
+        self._update_bar(gpu)
+        self._update_legend(gpu)
+        self._update_table(gpu)
+
+    def _update_stats(self, gpu: GPUInfo):
+        rate = gpu.idle_rate
+        # if rate <= 0.15:
+        #     free_style = "bold white on red"
+        # elif rate <= 0.5:
+        #     free_style = "bold black on yellow"
+        # else:
+        #     free_style = "bold black on green"
+        if rate <= 0.15:
+            free_style = "bold red"
+        elif rate <= 0.5:
+            free_style = "bold yellow"
+        else:
+            free_style = "bold green"
+
+        line = Text(no_wrap=True, overflow="ellipsis")
+        line.append("USED ", style="bold")
+        line.append(f"{fmt_gb(gpu.mem_used)}", style="bold")
+        line.append(f" / {fmt_gb(gpu.mem_total)} GB", style="dim")
+        line.append(f"  ({gpu.used_rate * 100:.0f}%)   ")
+        line.append(f" FREE {fmt_gb(gpu.mem_free)} GB ", style=free_style)
+        if gpu.util >= 0:
+            line.append(f"   ⚡ {gpu.util}%", style="cyan")
+        if gpu.temperature >= 0:
+            temp_style = "red" if gpu.temperature >= 80 else "cyan"
+            line.append(f"   🌡 {gpu.temperature}°C", style=temp_style)
+        self.stats.update(line)
+
+    def _update_bar(self, gpu: GPUInfo):
+        ordered = sorted(gpu.user_mems.items(), key=lambda kv: kv[1], reverse=True)
+        segments = [(self.monitor.color_for(u), float(m)) for u, m in ordered]
+        self.bar.set_data(segments, float(gpu.mem_total))
+
+    def _update_legend(self, gpu: GPUInfo):
+        ordered = sorted(gpu.user_mems.items(), key=lambda kv: kv[1], reverse=True)
+        legend = Text(no_wrap=True, overflow="ellipsis")
+        if not ordered:
+            legend.append("idle", style="dim")
+            self.legend.update(legend)
+            return
+        mvp = ordered[0][0]
+        for user, mem in ordered:
+            color = self.monitor.color_for(user)
+            pct = mem / gpu.mem_total * 100 if gpu.mem_total else 0
+            legend.append("● ", style=color)
+            crown = "🏆 " if user == mvp else ""
+            legend.append(f"{crown}{user} {fmt_gb(mem)}G {pct:.0f}%   ")
+        self.legend.update(legend)
+
+    def _update_table(self, gpu: GPUInfo) -> None:
+        # Rebuild columns each time so the sort arrow can move between headers.
+        self.table.clear(columns=True)
+        headers = self._header
+        labels = []
+        for i, (name, _) in enumerate(headers):
+            if i == self._sort_index:
+                name = f"{name} {'▼' if self._sort_reverse else '▲'}"
+            labels.append(name)
+        self.table.add_columns(*labels)
+
+        procs = list(gpu.procs)
+        if self._sort_index is not None:
+            key = headers[self._sort_index][1]
+            if key is not None:
+                procs.sort(key=key, reverse=self._sort_reverse)
+
+        for proc in procs:
+            color = self.monitor.color_for(proc.user)
+            row = [
+                str(proc.pid),
+                Text(f"● {proc.user}", style=color),
+                proc.number,
+                fmt_gb(proc.mem),
+                proc.started,
+                proc.runtime,
+                proc.cmd,
+            ]
+            if self.show_detail:
+                row.insert(-1, proc.detail)
+            self.table.add_row(*row)
+        if not procs:
+            empty = ["" for _ in headers]
+            empty[1] = "— no compute processes —"
+            self.table.add_row(*[Text(c, style="dim") for c in empty])

scopos-2.0.0.dist-info/METADATA

@@ -0,0 +1,106 @@
+Metadata-Version: 2.4
+Name: scopos
+Version: 2.0.0
+Summary: A Textual TUI for monitoring GPU memory usage, grouped by user.
+Author-email: Zhen Tian <zhen.tian.cs@gmail.com>
+Project-URL: Homepage, https://github.com/tinchen777/scopos-cli.git
+Project-URL: Repository, https://github.com/tinchen777/scopos-cli.git
+Project-URL: Issues, https://github.com/tinchen777/scopos-cli.git/issues
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: textual>=0.60
+Requires-Dist: psutil>=5.9
+Requires-Dist: nvidia-ml-py>=12.0
+Dynamic: license-file
+
+<div align="center">
+
+<h2 id="title">
+🐱‍👓 SCOPOS 🐱‍👓<br>
+<sub>NVIDIA GPU Monitor</sub>
+</h2>
+
+[![PyPI version](https://img.shields.io/pypi/v/scopos.svg)](https://pypi.org/project/scopos/)
+![Python](https://img.shields.io/pypi/pyversions/scopos?color=brightgreen)
+![License](https://img.shields.io/github/license/tinchen777/scopos.svg)
+
+![Github stars](https://img.shields.io/github/stars/tinchen777/scopos.svg)
+
+</div>
+
+```text
+  ___   ___  _____  ____  _____  ___
+ / __) / __)(  _  )(  _ \(  _  )/ __)
+ \__ \( (__  )(_)(  )___/ )(_)( \__ \
+ (___/ \___)(_____)(__)  (_____)(___/
+```
+
+## About
+
+Monitor NVIDIA GPU memory usage from the terminal, **grouped by user**. SCOPOS
+is built with [Textual](https://textual.textualize.io/): the layout adapts to
+your terminal size, and every GPU shows an at-a-glance bar of how its memory is
+split between users.
+
+- Python: 3.8+
+
+## Installation
+
+### Install with pipx
+
+`pipx` installs the application in an isolated environment while making
+the command globally available.
+
+```bash
+pip install pipx
+pipx ensurepath
+```
+
+```bash
+pipx install scopos
+```
+
+## Quick Start
+
+### monitor all GPUs
+
+```bash
+scopos
+```
+
+### highlight user "alice" and show their task details
+
+```bash
+scopos -u alice
+```
+
+### refresh every 2 seconds
+
+```bash
+scopos -i 2
+```
+
+### synthetic data, no NVIDIA driver needed
+
+```bash
+scopos --demo
+```
+
+---
+
+## Requirements
+
+- Python >= 3.8
+- `textual` >= 0.60
+- `psutil` >= 5.9
+- `nvidia-ml-py` >= 12.0
+
+## License
+
+See LICENSE in the repository.
+
+## Links
+
+- [Homepage/Repo](https://github.com/tinchen777/scopos.git)
+- [Issues](https://github.com/tinchen777/scopos.git/issues)

scopos-2.0.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+scopos/__init__.py,sha256=c7oK_575F5nfQE0HMLr24FNhpmusnLUp14oOo8J26QE,152
+scopos/app.py,sha256=x-u_-km_dpoN7oKyb5SqeuY3l4_uwGIiZeSSis9Dud4,5388
+scopos/cli.py,sha256=pQK8LHWZm3rzsMaS8aeBhLMdqrUGoNuzH3zDzAzXB88,1202
+scopos/monitor.py,sha256=b5m0LHBm-Jd6wm5XR248o96ILwEilcbhfNAknDdlNTA,14208
+scopos/widgets.py,sha256=YK4lgTzAfWNNyJy2ubbJlbudoS4FiUljYgkNHvrTZBE,10558
+scopos-2.0.0.dist-info/licenses/LICENSE,sha256=UB1vOlHpp1w6bfzyNq42Ou5m31wc-fPCX4YHR_pSxIw,1066
+scopos-2.0.0.dist-info/METADATA,sha256=Fx8QXWvine8NR95oHAzdmBvz0Nz1U008XbhK5M0_GZk,2235
+scopos-2.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+scopos-2.0.0.dist-info/entry_points.txt,sha256=SBfzJsiT9E9aBr8sgpirMJysEvk4q-kDoRYrhYC5sz4,43
+scopos-2.0.0.dist-info/top_level.txt,sha256=U6ZHTdL7oshtVFYHEjzx13othz6fdrW6_B0E0H_-3po,7
+scopos-2.0.0.dist-info/RECORD,,

scopos-2.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

scopos-2.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+scopos = scopos.cli:main

scopos-2.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Cathie Li
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scopos-2.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+scopos