gpuops 0.1.0__tar.gz

gpuops-0.1.0/PKG-INFO

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: gpuops
+Version: 0.1.0
+Summary: A small GPU operations CLI for shared lab servers.
+Author: GPUOps contributors
+License-Expression: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# GPUOps
+
+GPUOps is a small CLI for managing shared lab GPUs. It focuses on the first
+things an admin usually needs: seeing who is using GPUs, finding suspicious
+idle jobs, safely killing selected GPU processes, and keeping lightweight usage
+history.
+
+## Features
+
+- Real-time GPU status: memory, utilization, temperature, power, and processes.
+- GPU process list with user, command, working directory, and running time.
+- Kill all processes on a GPU, all GPU processes for a user, or that user on one GPU.
+- Sorting by free memory, used memory, utilization, temperature, or GPU index.
+- Stalled-GPU warning when memory is occupied but utilization stays near zero.
+- Per-user summary across all GPUs.
+- Memory/running-time top view.
+- Lightweight JSONL history sampling and GPU-hour aggregation.
+- Safe-by-default process management: `kill` is dry-run unless `--yes` is passed.
+
+## Install locally
+
+```bash
+python3 -m pip install -e .
+```
+
+## Commands
+
+```bash
+gpuops status
+gpuops status --sort free --record
+gpuops ps --gpu 0
+gpuops users
+gpuops top --by memory
+gpuops doctor
+gpuops history --days 30
+```
+
+Kill commands are dry-run by default:
+
+```bash
+gpuops kill --gpu 3
+gpuops kill --user alice
+gpuops kill --gpu 3 --user alice
+```
+
+To actually send signals:
+
+```bash
+gpuops kill --gpu 3 --user alice --yes
+gpuops kill --user alice --force --yes
+```
+
+## History
+
+`gpuops status --record` appends one sample to:
+
+```text
+~/.local/share/gpuops/history.jsonl
+```
+
+Run it from cron or systemd every minute if you want durable usage stats.
+
+## Notes
+
+GPUOps uses `nvidia-smi` when available and enriches process information from
+`/proc` on Linux. On machines without NVIDIA GPUs, commands fail cleanly with a
+message instead of crashing.

gpuops-0.1.0/README.md

@@ -0,0 +1,67 @@
+# GPUOps
+
+GPUOps is a small CLI for managing shared lab GPUs. It focuses on the first
+things an admin usually needs: seeing who is using GPUs, finding suspicious
+idle jobs, safely killing selected GPU processes, and keeping lightweight usage
+history.
+
+## Features
+
+- Real-time GPU status: memory, utilization, temperature, power, and processes.
+- GPU process list with user, command, working directory, and running time.
+- Kill all processes on a GPU, all GPU processes for a user, or that user on one GPU.
+- Sorting by free memory, used memory, utilization, temperature, or GPU index.
+- Stalled-GPU warning when memory is occupied but utilization stays near zero.
+- Per-user summary across all GPUs.
+- Memory/running-time top view.
+- Lightweight JSONL history sampling and GPU-hour aggregation.
+- Safe-by-default process management: `kill` is dry-run unless `--yes` is passed.
+
+## Install locally
+
+```bash
+python3 -m pip install -e .
+```
+
+## Commands
+
+```bash
+gpuops status
+gpuops status --sort free --record
+gpuops ps --gpu 0
+gpuops users
+gpuops top --by memory
+gpuops doctor
+gpuops history --days 30
+```
+
+Kill commands are dry-run by default:
+
+```bash
+gpuops kill --gpu 3
+gpuops kill --user alice
+gpuops kill --gpu 3 --user alice
+```
+
+To actually send signals:
+
+```bash
+gpuops kill --gpu 3 --user alice --yes
+gpuops kill --user alice --force --yes
+```
+
+## History
+
+`gpuops status --record` appends one sample to:
+
+```text
+~/.local/share/gpuops/history.jsonl
+```
+
+Run it from cron or systemd every minute if you want durable usage stats.
+
+## Notes
+
+GPUOps uses `nvidia-smi` when available and enriches process information from
+`/proc` on Linux. On machines without NVIDIA GPUs, commands fail cleanly with a
+message instead of crashing.

gpuops-0.1.0/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "gpuops"
+version = "0.1.0"
+description = "A small GPU operations CLI for shared lab servers."
+readme = "README.md"
+requires-python = ">=3.9"
+license = "MIT"
+authors = [{ name = "GPUOps contributors" }]
+dependencies = []
+
+[project.scripts]
+gpuops = "gpuops.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]

gpuops-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

gpuops-0.1.0/src/gpuops/__init__.py

@@ -0,0 +1,5 @@
+"""GPUOps package."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.1.0"

gpuops-0.1.0/src/gpuops/__main__.py

@@ -0,0 +1,5 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+

gpuops-0.1.0/src/gpuops/actions.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import os
+import signal
+from dataclasses import dataclass
+from typing import Iterable, List, Optional
+
+from .models import GPUProcess
+
+
+@dataclass(frozen=True)
+class KillResult:
+    pid: int
+    user: Optional[str]
+    gpu_index: Optional[int]
+    signal_name: str
+    killed: bool
+    message: str
+
+
+def filter_processes(processes: Iterable[GPUProcess], gpu: Optional[int] = None, user: Optional[str] = None) -> List[GPUProcess]:
+    selected = []
+    for proc in processes:
+        if gpu is not None and proc.gpu_index != gpu:
+            continue
+        if user is not None and proc.user != user:
+            continue
+        selected.append(proc)
+    return selected
+
+
+def signal_for(force: bool) -> signal.Signals:
+    return signal.SIGKILL if force else signal.SIGTERM
+
+
+def kill_processes(processes: Iterable[GPUProcess], sig: signal.Signals, dry_run: bool = True) -> List[KillResult]:
+    results: List[KillResult] = []
+    for proc in processes:
+        if dry_run:
+            results.append(
+                KillResult(
+                    pid=proc.pid,
+                    user=proc.user,
+                    gpu_index=proc.gpu_index,
+                    signal_name=sig.name,
+                    killed=False,
+                    message="dry-run",
+                )
+            )
+            continue
+        try:
+            os.kill(proc.pid, sig)
+            results.append(
+                KillResult(
+                    pid=proc.pid,
+                    user=proc.user,
+                    gpu_index=proc.gpu_index,
+                    signal_name=sig.name,
+                    killed=True,
+                    message="signal sent",
+                )
+            )
+        except ProcessLookupError:
+            results.append(
+                KillResult(
+                    pid=proc.pid,
+                    user=proc.user,
+                    gpu_index=proc.gpu_index,
+                    signal_name=sig.name,
+                    killed=False,
+                    message="process already exited",
+                )
+            )
+        except PermissionError:
+            results.append(
+                KillResult(
+                    pid=proc.pid,
+                    user=proc.user,
+                    gpu_index=proc.gpu_index,
+                    signal_name=sig.name,
+                    killed=False,
+                    message="permission denied",
+                )
+            )
+    return results
+

gpuops-0.1.0/src/gpuops/alerts.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List
+
+from .models import GPU, Snapshot
+
+
+@dataclass(frozen=True)
+class Alert:
+    gpu_index: int
+    level: str
+    message: str
+
+
+def evaluate_snapshot(
+    snapshot: Snapshot,
+    idle_util_percent: int = 5,
+    occupied_memory_mb: int = 1024,
+    high_temp_c: int = 82,
+) -> List[Alert]:
+    alerts: List[Alert] = []
+    process_by_gpu = {}
+    for proc in snapshot.processes:
+        if proc.gpu_index is not None:
+            process_by_gpu.setdefault(proc.gpu_index, 0)
+            process_by_gpu[proc.gpu_index] += 1
+
+    for gpu in snapshot.gpus:
+        alerts.extend(_gpu_alerts(gpu, process_by_gpu.get(gpu.index, 0), idle_util_percent, occupied_memory_mb, high_temp_c))
+    return alerts
+
+
+def _gpu_alerts(
+    gpu: GPU,
+    process_count: int,
+    idle_util_percent: int,
+    occupied_memory_mb: int,
+    high_temp_c: int,
+) -> List[Alert]:
+    alerts: List[Alert] = []
+    if gpu.memory_used_mb >= occupied_memory_mb and gpu.utilization_gpu_percent <= idle_util_percent:
+        alerts.append(
+            Alert(
+                gpu_index=gpu.index,
+                level="warn",
+                message="memory occupied while utilization is near zero",
+            )
+        )
+    if gpu.memory_used_mb >= occupied_memory_mb and process_count == 0:
+        alerts.append(
+            Alert(
+                gpu_index=gpu.index,
+                level="warn",
+                message="memory occupied but nvidia-smi reports no compute process",
+            )
+        )
+    if gpu.temperature_c is not None and gpu.temperature_c >= high_temp_c:
+        alerts.append(Alert(gpu_index=gpu.index, level="crit", message=f"temperature is high ({gpu.temperature_c}C)"))
+    return alerts
+

gpuops-0.1.0/src/gpuops/cli.py

@@ -0,0 +1,226 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import List, Optional
+
+from .actions import filter_processes, kill_processes, signal_for
+from .alerts import evaluate_snapshot
+from .formatting import gpu_rows, json_dump, process_rows, table
+from .history import DEFAULT_HISTORY_PATH, read_snapshots, record_snapshot, since_days, summarize_user_usage
+from .nvidia import NvidiaSmiError, collect_snapshot
+from .process_info import enrich_processes
+from .models import Snapshot
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="gpuops", description="GPUOps shared GPU administration CLI")
+    parser.add_argument("--json", action="store_true", help="print machine-readable JSON")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    status = subparsers.add_parser("status", help="show real-time GPU status")
+    status.add_argument("--json", action="store_true", help="print machine-readable JSON")
+    status.add_argument("--sort", choices=["index", "free", "memory", "util", "temp"], default="index")
+    status.add_argument("--record", action="store_true", help="append this snapshot to the history file")
+
+    ps = subparsers.add_parser("ps", help="show GPU processes")
+    ps.add_argument("--json", action="store_true", help="print machine-readable JSON")
+    ps.add_argument("--gpu", type=int)
+    ps.add_argument("--user")
+
+    users = subparsers.add_parser("users", help="summarize current usage by user")
+    users.add_argument("--json", action="store_true", help="print machine-readable JSON")
+    users.add_argument("--sort", choices=["memory", "processes", "runtime"], default="memory")
+
+    top = subparsers.add_parser("top", help="rank GPU processes")
+    top.add_argument("--json", action="store_true", help="print machine-readable JSON")
+    top.add_argument("--by", choices=["memory", "runtime"], default="memory")
+    top.add_argument("--limit", type=int, default=20)
+
+    doctor = subparsers.add_parser("doctor", help="show suspicious GPU states")
+    doctor.add_argument("--json", action="store_true", help="print machine-readable JSON")
+    doctor.add_argument("--idle-util", type=int, default=5)
+    doctor.add_argument("--occupied-mb", type=int, default=1024)
+
+    kill = subparsers.add_parser("kill", help="kill selected GPU processes; dry-run unless --yes is passed")
+    kill.add_argument("--json", action="store_true", help="print machine-readable JSON")
+    kill.add_argument("--gpu", type=int)
+    kill.add_argument("--user")
+    kill.add_argument("--force", action="store_true", help="use SIGKILL instead of SIGTERM")
+    kill.add_argument("--yes", action="store_true", help="actually send the signal")
+
+    history = subparsers.add_parser("history", help="summarize recorded GPU usage")
+    history.add_argument("--json", action="store_true", help="print machine-readable JSON")
+    history.add_argument("--days", type=int, default=30)
+    history.add_argument("--path", default=str(DEFAULT_HISTORY_PATH))
+
+    return parser
+
+
+def _snapshot() -> Snapshot:
+    snap = collect_snapshot()
+    enrich_processes(snap.processes)
+    return snap
+
+
+def _sort_gpus(gpus, key: str):
+    if key == "free":
+        return sorted(gpus, key=lambda gpu: gpu.memory_free_mb, reverse=True)
+    if key == "memory":
+        return sorted(gpus, key=lambda gpu: gpu.memory_used_mb, reverse=True)
+    if key == "util":
+        return sorted(gpus, key=lambda gpu: gpu.utilization_gpu_percent, reverse=True)
+    if key == "temp":
+        return sorted(gpus, key=lambda gpu: gpu.temperature_c or -1, reverse=True)
+    return sorted(gpus, key=lambda gpu: gpu.index)
+
+
+def cmd_status(args) -> int:
+    snap = _snapshot()
+    if args.record:
+        record_snapshot(snap)
+    alerts = evaluate_snapshot(snap)
+    gpus = _sort_gpus(snap.gpus, args.sort)
+    if args.json:
+        print(json_dump({"snapshot": snap.to_dict(), "alerts": [alert.__dict__ for alert in alerts]}))
+        return 0
+    print(table(["GPU", "Name", "Memory", "Free", "Util", "Temp", "Power", "Alerts"], gpu_rows(gpus, alerts)))
+    return 0
+
+
+def cmd_ps(args) -> int:
+    snap = _snapshot()
+    processes = filter_processes(snap.processes, gpu=args.gpu, user=args.user)
+    processes.sort(key=lambda proc: (proc.gpu_index if proc.gpu_index is not None else 9999, -proc.used_memory_mb))
+    if args.json:
+        print(json_dump([proc.__dict__ for proc in processes]))
+        return 0
+    print(table(["GPU", "PID", "User", "Memory", "Running", "Command", "CWD"], process_rows(processes)))
+    return 0
+
+
+def cmd_users(args) -> int:
+    snap = _snapshot()
+    buckets = defaultdict(lambda: {"memory": 0, "processes": 0, "runtime": 0.0, "gpus": set()})
+    for proc in snap.processes:
+        user = proc.user or "unknown"
+        buckets[user]["memory"] += proc.used_memory_mb
+        buckets[user]["processes"] += 1
+        buckets[user]["runtime"] += proc.running_seconds or 0.0
+        if proc.gpu_index is not None:
+            buckets[user]["gpus"].add(proc.gpu_index)
+    summaries = []
+    for user, values in buckets.items():
+        summaries.append(
+            {
+                "user": user,
+                "gpus": ",".join(str(gpu) for gpu in sorted(values["gpus"])) or "-",
+                "processes": values["processes"],
+                "memory_mb": values["memory"],
+                "runtime_seconds": values["runtime"],
+            }
+        )
+    if args.sort == "memory":
+        summaries.sort(key=lambda item: item["memory_mb"], reverse=True)
+    elif args.sort == "processes":
+        summaries.sort(key=lambda item: item["processes"], reverse=True)
+    else:
+        summaries.sort(key=lambda item: item["runtime_seconds"], reverse=True)
+    if args.json:
+        print(json_dump(summaries))
+        return 0
+    rows = [
+        [item["user"], item["gpus"], item["processes"], f"{item['memory_mb']} MB", _duration(item["runtime_seconds"])]
+        for item in summaries
+    ]
+    print(table(["User", "GPUs", "Processes", "Memory", "Total Running"], rows))
+    return 0
+
+
+def _duration(seconds: float) -> str:
+    from .formatting import human_duration
+
+    return human_duration(seconds)
+
+
+def cmd_top(args) -> int:
+    snap = _snapshot()
+    if args.by == "runtime":
+        processes = sorted(snap.processes, key=lambda proc: proc.running_seconds or 0, reverse=True)
+    else:
+        processes = sorted(snap.processes, key=lambda proc: proc.used_memory_mb, reverse=True)
+    processes = processes[: max(0, args.limit)]
+    if args.json:
+        print(json_dump([proc.__dict__ for proc in processes]))
+        return 0
+    print(table(["GPU", "PID", "User", "Memory", "Running", "Command", "CWD"], process_rows(processes)))
+    return 0
+
+
+def cmd_doctor(args) -> int:
+    snap = _snapshot()
+    alerts = evaluate_snapshot(snap, idle_util_percent=args.idle_util, occupied_memory_mb=args.occupied_mb)
+    if args.json:
+        print(json_dump([alert.__dict__ for alert in alerts]))
+        return 0
+    if not alerts:
+        print("No suspicious GPU state found.")
+        return 0
+    rows = [[alert.level, alert.gpu_index, alert.message] for alert in alerts]
+    print(table(["Level", "GPU", "Message"], rows))
+    return 1 if any(alert.level == "crit" for alert in alerts) else 0
+
+
+def cmd_kill(args) -> int:
+    if args.gpu is None and args.user is None:
+        print("Refusing to select every GPU process. Pass --gpu, --user, or both.", file=sys.stderr)
+        return 2
+    snap = _snapshot()
+    selected = filter_processes(snap.processes, gpu=args.gpu, user=args.user)
+    sig = signal_for(args.force)
+    results = kill_processes(selected, sig=sig, dry_run=not args.yes)
+    rows = [[r.gpu_index if r.gpu_index is not None else "-", r.pid, r.user or "-", r.signal_name, r.message] for r in results]
+    if args.json:
+        print(json_dump([result.__dict__ for result in results]))
+        return 0
+    if not rows:
+        print("No matching GPU processes.")
+        return 0
+    print(table(["GPU", "PID", "User", "Signal", "Result"], rows))
+    if not args.yes:
+        print("\nDry-run only. Re-run with --yes to send the signal.")
+    return 0
+
+
+def cmd_history(args) -> int:
+    samples = read_snapshots(path=Path(args.path).expanduser(), since=since_days(args.days))
+    usage = summarize_user_usage(samples)
+    if args.json:
+        print(json_dump([item.__dict__ for item in usage]))
+        return 0
+    if not usage:
+        print("No history samples found. Run `gpuops status --record` periodically first.")
+        return 0
+    rows = [[item.user, f"{item.gpu_hours:.2f}", f"{item.memory_gb_hours:.2f}", item.samples] for item in usage]
+    print(table(["User", "GPU Hours", "GB Hours", "Samples"], rows))
+    return 0
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return {
+            "status": cmd_status,
+            "ps": cmd_ps,
+            "users": cmd_users,
+            "top": cmd_top,
+            "doctor": cmd_doctor,
+            "kill": cmd_kill,
+            "history": cmd_history,
+        }[args.command](args)
+    except NvidiaSmiError as exc:
+        print(str(exc), file=sys.stderr)
+        return 127

gpuops-0.1.0/src/gpuops/formatting.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import json
+from typing import Iterable, List, Sequence
+
+from .alerts import Alert
+from .models import GPU, GPUProcess
+
+
+def human_duration(seconds: float | None) -> str:
+    if seconds is None:
+        return "-"
+    seconds = int(seconds)
+    days, seconds = divmod(seconds, 86400)
+    hours, seconds = divmod(seconds, 3600)
+    minutes, _ = divmod(seconds, 60)
+    if days:
+        return f"{days}d{hours}h"
+    if hours:
+        return f"{hours}h{minutes}m"
+    return f"{minutes}m"
+
+
+def json_dump(data: object) -> str:
+    return json.dumps(data, indent=2, sort_keys=True)
+
+
+def table(headers: Sequence[str], rows: Iterable[Sequence[object]]) -> str:
+    rows = [[str(cell) for cell in row] for row in rows]
+    widths: List[int] = [len(header) for header in headers]
+    for row in rows:
+        for index, cell in enumerate(row):
+            widths[index] = max(widths[index], len(cell))
+    line = "  ".join(header.ljust(widths[index]) for index, header in enumerate(headers))
+    sep = "  ".join("-" * width for width in widths)
+    body = ["  ".join(cell.ljust(widths[index]) for index, cell in enumerate(row)) for row in rows]
+    return "\n".join([line, sep, *body])
+
+
+def gpu_rows(gpus: Sequence[GPU], alerts: Sequence[Alert]) -> List[List[object]]:
+    alert_by_gpu = {}
+    for alert in alerts:
+        alert_by_gpu.setdefault(alert.gpu_index, []).append(alert.message)
+    return [
+        [
+            gpu.index,
+            gpu.name,
+            f"{gpu.memory_used_mb}/{gpu.memory_total_mb} MB",
+            f"{gpu.memory_free_mb} MB",
+            f"{gpu.utilization_gpu_percent}%",
+            "-" if gpu.temperature_c is None else f"{gpu.temperature_c}C",
+            "-" if gpu.power_w is None else f"{gpu.power_w:.0f}W",
+            "; ".join(alert_by_gpu.get(gpu.index, [])) or "-",
+        ]
+        for gpu in gpus
+    ]
+
+
+def process_rows(processes: Sequence[GPUProcess]) -> List[List[object]]:
+    return [
+        [
+            proc.gpu_index if proc.gpu_index is not None else "-",
+            proc.pid,
+            proc.user or "-",
+            f"{proc.used_memory_mb} MB",
+            human_duration(proc.running_seconds),
+            proc.command or proc.process_name,
+            proc.cwd or "-",
+        ]
+        for proc in processes
+    ]
+

gpuops-0.1.0/src/gpuops/history.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import json
+import os
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional
+
+from .models import Snapshot
+
+
+DEFAULT_HISTORY_PATH = Path(os.environ.get("GPUOPS_HISTORY", "~/.local/share/gpuops/history.jsonl")).expanduser()
+
+
+@dataclass(frozen=True)
+class UserUsage:
+    user: str
+    gpu_hours: float
+    memory_gb_hours: float
+    samples: int
+
+
+def record_snapshot(snapshot: Snapshot, path: Path = DEFAULT_HISTORY_PATH) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(snapshot.to_dict(), sort_keys=True) + "\n")
+
+
+def read_snapshots(path: Path = DEFAULT_HISTORY_PATH, since: Optional[float] = None) -> List[dict]:
+    if not path.exists():
+        return []
+    samples: List[dict] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                sample = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if since is None or float(sample.get("timestamp", 0)) >= since:
+                samples.append(sample)
+    samples.sort(key=lambda item: float(item.get("timestamp", 0)))
+    return samples
+
+
+def summarize_user_usage(samples: Iterable[dict], max_interval_seconds: int = 300) -> List[UserUsage]:
+    ordered = sorted(samples, key=lambda item: float(item.get("timestamp", 0)))
+    accum: Dict[str, Dict[str, float]] = {}
+    for current, nxt in zip(ordered, ordered[1:]):
+        timestamp = float(current.get("timestamp", 0))
+        next_timestamp = float(nxt.get("timestamp", timestamp))
+        interval = max(0.0, min(max_interval_seconds, next_timestamp - timestamp))
+        if interval <= 0:
+            continue
+        seen_user_gpu = set()
+        for proc in current.get("processes", []):
+            user = proc.get("user") or "unknown"
+            gpu_index = proc.get("gpu_index")
+            used_memory_mb = float(proc.get("used_memory_mb") or 0)
+            bucket = accum.setdefault(user, {"gpu_seconds": 0.0, "memory_mb_seconds": 0.0, "samples": 0.0})
+            key = (user, gpu_index)
+            if key not in seen_user_gpu:
+                bucket["gpu_seconds"] += interval
+                seen_user_gpu.add(key)
+            bucket["memory_mb_seconds"] += used_memory_mb * interval
+            bucket["samples"] += 1
+
+    return sorted(
+        [
+            UserUsage(
+                user=user,
+                gpu_hours=values["gpu_seconds"] / 3600.0,
+                memory_gb_hours=values["memory_mb_seconds"] / 1024.0 / 3600.0,
+                samples=int(values["samples"]),
+            )
+            for user, values in accum.items()
+        ],
+        key=lambda item: item.gpu_hours,
+        reverse=True,
+    )
+
+
+def since_days(days: int) -> float:
+    return time.time() - (days * 86400)

gpuops-0.1.0/src/gpuops/models.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass(frozen=True)
+class GPU:
+    index: int
+    uuid: str
+    name: str
+    memory_total_mb: int
+    memory_used_mb: int
+    utilization_gpu_percent: int
+    temperature_c: Optional[int]
+    power_w: Optional[float]
+
+    @property
+    def memory_free_mb(self) -> int:
+        return max(0, self.memory_total_mb - self.memory_used_mb)
+
+
+@dataclass
+class GPUProcess:
+    pid: int
+    gpu_uuid: str
+    gpu_index: Optional[int]
+    used_memory_mb: int
+    process_name: str
+    user: Optional[str] = None
+    command: Optional[str] = None
+    cwd: Optional[str] = None
+    started_at: Optional[float] = None
+    running_seconds: Optional[float] = None
+
+
+@dataclass
+class Snapshot:
+    timestamp: float
+    gpus: List[GPU] = field(default_factory=list)
+    processes: List[GPUProcess] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "timestamp": self.timestamp,
+            "gpus": [gpu.__dict__ for gpu in self.gpus],
+            "processes": [proc.__dict__ for proc in self.processes],
+        }
+

gpuops-0.1.0/src/gpuops/nvidia.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import csv
+import subprocess
+import time
+from io import StringIO
+from typing import Callable, Iterable, List, Optional
+
+from .models import GPU, GPUProcess, Snapshot
+
+
+class NvidiaSmiError(RuntimeError):
+    pass
+
+
+Runner = Callable[[List[str]], str]
+
+
+def _default_runner(args: List[str]) -> str:
+    try:
+        result = subprocess.run(
+            ["nvidia-smi", *args],
+            check=True,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+    except FileNotFoundError as exc:
+        raise NvidiaSmiError("nvidia-smi was not found on this machine") from exc
+    except subprocess.CalledProcessError as exc:
+        message = exc.stderr.strip() or exc.stdout.strip() or str(exc)
+        raise NvidiaSmiError(f"nvidia-smi failed: {message}") from exc
+    return result.stdout
+
+
+def _rows(text: str) -> Iterable[List[str]]:
+    reader = csv.reader(StringIO(text))
+    for row in reader:
+        cleaned = [cell.strip() for cell in row]
+        if cleaned and any(cell for cell in cleaned):
+            yield cleaned
+
+
+def _int(value: str, default: int = 0) -> int:
+    value = value.strip()
+    if not value or value.upper() == "[N/A]" or value == "N/A":
+        return default
+    return int(float(value))
+
+
+def _float_or_none(value: str) -> Optional[float]:
+    value = value.strip()
+    if not value or value.upper() == "[N/A]" or value == "N/A":
+        return None
+    return float(value)
+
+
+def query_gpus(runner: Runner = _default_runner) -> List[GPU]:
+    output = runner(
+        [
+            "--query-gpu=index,uuid,name,memory.total,memory.used,utilization.gpu,temperature.gpu,power.draw",
+            "--format=csv,noheader,nounits",
+        ]
+    )
+    gpus: List[GPU] = []
+    for row in _rows(output):
+        if len(row) < 8:
+            continue
+        gpus.append(
+            GPU(
+                index=_int(row[0]),
+                uuid=row[1],
+                name=row[2],
+                memory_total_mb=_int(row[3]),
+                memory_used_mb=_int(row[4]),
+                utilization_gpu_percent=_int(row[5]),
+                temperature_c=None if row[6] in {"", "N/A", "[N/A]"} else _int(row[6]),
+                power_w=_float_or_none(row[7]),
+            )
+        )
+    return gpus
+
+
+def query_processes(runner: Runner = _default_runner) -> List[GPUProcess]:
+    try:
+        output = runner(
+            [
+                "--query-compute-apps=pid,gpu_uuid,used_gpu_memory,process_name",
+                "--format=csv,noheader,nounits",
+            ]
+        )
+    except NvidiaSmiError as exc:
+        text = str(exc).lower()
+        if "not supported" in text or "no running processes" in text:
+            return []
+        raise
+
+    processes: List[GPUProcess] = []
+    for row in _rows(output):
+        if len(row) < 4 or row[0] in {"", "N/A", "[Not Supported]"}:
+            continue
+        processes.append(
+            GPUProcess(
+                pid=_int(row[0]),
+                gpu_uuid=row[1],
+                gpu_index=None,
+                used_memory_mb=_int(row[2]),
+                process_name=row[3],
+            )
+        )
+    return processes
+
+
+def collect_snapshot(runner: Runner = _default_runner) -> Snapshot:
+    gpus = query_gpus(runner)
+    uuid_to_index = {gpu.uuid: gpu.index for gpu in gpus}
+    processes = query_processes(runner)
+    for proc in processes:
+        proc.gpu_index = uuid_to_index.get(proc.gpu_uuid)
+    return Snapshot(timestamp=time.time(), gpus=gpus, processes=processes)
+

gpuops-0.1.0/src/gpuops/process_info.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import os
+import pwd
+import time
+from typing import Iterable, Optional
+
+from .models import GPUProcess
+
+
+def _boot_time() -> Optional[float]:
+    try:
+        with open("/proc/stat", "r", encoding="utf-8") as handle:
+            for line in handle:
+                if line.startswith("btime "):
+                    return float(line.split()[1])
+    except OSError:
+        return None
+    return None
+
+
+def _clock_ticks() -> int:
+    try:
+        return os.sysconf(os.sysconf_names["SC_CLK_TCK"])
+    except (KeyError, ValueError, OSError):
+        return 100
+
+
+def enrich_process(proc: GPUProcess, now: Optional[float] = None) -> GPUProcess:
+    now = time.time() if now is None else now
+    proc_dir = f"/proc/{proc.pid}"
+
+    try:
+        stat_info = os.stat(proc_dir)
+        proc.user = pwd.getpwuid(stat_info.st_uid).pw_name
+    except (OSError, KeyError):
+        pass
+
+    try:
+        with open(f"{proc_dir}/cmdline", "rb") as handle:
+            raw = handle.read().replace(b"\x00", b" ").strip()
+        if raw:
+            proc.command = raw.decode("utf-8", errors="replace")
+    except OSError:
+        pass
+
+    try:
+        proc.cwd = os.readlink(f"{proc_dir}/cwd")
+    except OSError:
+        pass
+
+    boot = _boot_time()
+    if boot is not None:
+        try:
+            with open(f"{proc_dir}/stat", "r", encoding="utf-8") as handle:
+                stat = handle.read()
+            parts = stat.rsplit(") ", 1)[1].split()
+            start_ticks = int(parts[19])
+            proc.started_at = boot + (start_ticks / _clock_ticks())
+            proc.running_seconds = max(0.0, now - proc.started_at)
+        except (OSError, ValueError, IndexError):
+            pass
+
+    if not proc.command:
+        proc.command = proc.process_name
+    return proc
+
+
+def enrich_processes(processes: Iterable[GPUProcess]) -> None:
+    now = time.time()
+    for proc in processes:
+        enrich_process(proc, now=now)
+

gpuops-0.1.0/src/gpuops.egg-info/PKG-INFO

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: gpuops
+Version: 0.1.0
+Summary: A small GPU operations CLI for shared lab servers.
+Author: GPUOps contributors
+License-Expression: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# GPUOps
+
+GPUOps is a small CLI for managing shared lab GPUs. It focuses on the first
+things an admin usually needs: seeing who is using GPUs, finding suspicious
+idle jobs, safely killing selected GPU processes, and keeping lightweight usage
+history.
+
+## Features
+
+- Real-time GPU status: memory, utilization, temperature, power, and processes.
+- GPU process list with user, command, working directory, and running time.
+- Kill all processes on a GPU, all GPU processes for a user, or that user on one GPU.
+- Sorting by free memory, used memory, utilization, temperature, or GPU index.
+- Stalled-GPU warning when memory is occupied but utilization stays near zero.
+- Per-user summary across all GPUs.
+- Memory/running-time top view.
+- Lightweight JSONL history sampling and GPU-hour aggregation.
+- Safe-by-default process management: `kill` is dry-run unless `--yes` is passed.
+
+## Install locally
+
+```bash
+python3 -m pip install -e .
+```
+
+## Commands
+
+```bash
+gpuops status
+gpuops status --sort free --record
+gpuops ps --gpu 0
+gpuops users
+gpuops top --by memory
+gpuops doctor
+gpuops history --days 30
+```
+
+Kill commands are dry-run by default:
+
+```bash
+gpuops kill --gpu 3
+gpuops kill --user alice
+gpuops kill --gpu 3 --user alice
+```
+
+To actually send signals:
+
+```bash
+gpuops kill --gpu 3 --user alice --yes
+gpuops kill --user alice --force --yes
+```
+
+## History
+
+`gpuops status --record` appends one sample to:
+
+```text
+~/.local/share/gpuops/history.jsonl
+```
+
+Run it from cron or systemd every minute if you want durable usage stats.
+
+## Notes
+
+GPUOps uses `nvidia-smi` when available and enriches process information from
+`/proc` on Linux. On machines without NVIDIA GPUs, commands fail cleanly with a
+message instead of crashing.

gpuops-0.1.0/src/gpuops.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+README.md
+pyproject.toml
+src/gpuops/__init__.py
+src/gpuops/__main__.py
+src/gpuops/actions.py
+src/gpuops/alerts.py
+src/gpuops/cli.py
+src/gpuops/formatting.py
+src/gpuops/history.py
+src/gpuops/models.py
+src/gpuops/nvidia.py
+src/gpuops/process_info.py
+src/gpuops.egg-info/PKG-INFO
+src/gpuops.egg-info/SOURCES.txt
+src/gpuops.egg-info/dependency_links.txt
+src/gpuops.egg-info/entry_points.txt
+src/gpuops.egg-info/top_level.txt
+tests/test_alerts_history_actions.py
+tests/test_nvidia.py

gpuops-0.1.0/src/gpuops.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

gpuops-0.1.0/src/gpuops.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+gpuops = gpuops.cli:main

gpuops-0.1.0/src/gpuops.egg-info/top_level.txt

@@ -0,0 +1 @@
+gpuops

gpuops-0.1.0/tests/test_alerts_history_actions.py

@@ -0,0 +1,57 @@
+import signal
+import unittest
+
+from gpuops.actions import filter_processes, kill_processes
+from gpuops.alerts import evaluate_snapshot
+from gpuops.history import summarize_user_usage
+from gpuops.models import GPU, GPUProcess, Snapshot
+
+
+class AlertsHistoryActionsTests(unittest.TestCase):
+    def test_alerts_idle_occupied_gpu(self):
+        snapshot = Snapshot(
+            timestamp=1,
+            gpus=[GPU(0, "GPU-a", "A100", 80000, 70000, 0, 74, 220.0)],
+            processes=[GPUProcess(10, "GPU-a", 0, 70000, "python", user="alice")],
+        )
+
+        alerts = evaluate_snapshot(snapshot)
+
+        self.assertEqual(alerts[0].gpu_index, 0)
+        self.assertIn("near zero", alerts[0].message)
+
+    def test_history_counts_gpu_hours_once_per_user_gpu(self):
+        samples = [
+            {
+                "timestamp": 0,
+                "processes": [
+                    {"user": "alice", "gpu_index": 0, "used_memory_mb": 1024},
+                    {"user": "alice", "gpu_index": 0, "used_memory_mb": 2048},
+                    {"user": "bob", "gpu_index": 1, "used_memory_mb": 1024},
+                ],
+            },
+            {"timestamp": 3600, "processes": []},
+        ]
+
+        usage = summarize_user_usage(samples, max_interval_seconds=3600)
+
+        self.assertEqual(usage[0].user, "alice")
+        self.assertAlmostEqual(usage[0].gpu_hours, 1.0)
+        self.assertAlmostEqual(usage[0].memory_gb_hours, 3.0)
+
+    def test_filter_and_dry_run_kill(self):
+        processes = [
+            GPUProcess(10, "GPU-a", 0, 100, "python", user="alice"),
+            GPUProcess(11, "GPU-b", 1, 100, "python", user="bob"),
+        ]
+
+        selected = filter_processes(processes, gpu=0, user="alice")
+        results = kill_processes(selected, signal.SIGTERM, dry_run=True)
+
+        self.assertEqual([proc.pid for proc in selected], [10])
+        self.assertFalse(results[0].killed)
+        self.assertEqual(results[0].message, "dry-run")
+
+
+if __name__ == "__main__":
+    unittest.main()

gpuops-0.1.0/tests/test_nvidia.py

@@ -0,0 +1,25 @@
+import unittest
+
+from gpuops.nvidia import collect_snapshot
+
+
+class NvidiaParsingTests(unittest.TestCase):
+    def test_collect_snapshot_maps_processes_to_gpu_index(self):
+        def runner(args):
+            query = args[0]
+            if query.startswith("--query-gpu"):
+                return "0, GPU-a, NVIDIA A100, 81920, 40960, 0, 70, 250.5\n1, GPU-b, NVIDIA A100, 81920, 0, 0, 35, [N/A]\n"
+            if query.startswith("--query-compute-apps"):
+                return "123, GPU-a, 20480, python\n"
+            raise AssertionError(args)
+
+        snapshot = collect_snapshot(runner)
+
+        self.assertEqual(len(snapshot.gpus), 2)
+        self.assertEqual(snapshot.gpus[0].memory_free_mb, 40960)
+        self.assertEqual(snapshot.processes[0].gpu_index, 0)
+        self.assertEqual(snapshot.processes[0].used_memory_mb, 20480)
+
+
+if __name__ == "__main__":
+    unittest.main()