gpuops 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gpuops-0.1.0 → gpuops-0.1.1}/PKG-INFO +13 -1
- {gpuops-0.1.0 → gpuops-0.1.1}/README.md +11 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/pyproject.toml +2 -2
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/__init__.py +1 -1
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/cli.py +70 -46
- gpuops-0.1.1/src/gpuops/ui.py +206 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops.egg-info/PKG-INFO +13 -1
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops.egg-info/SOURCES.txt +2 -0
- gpuops-0.1.1/src/gpuops.egg-info/requires.txt +1 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/setup.cfg +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/__main__.py +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/actions.py +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/alerts.py +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/formatting.py +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/history.py +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/models.py +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/nvidia.py +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops/process_info.py +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops.egg-info/dependency_links.txt +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops.egg-info/entry_points.txt +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/src/gpuops.egg-info/top_level.txt +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/tests/test_alerts_history_actions.py +0 -0
- {gpuops-0.1.0 → gpuops-0.1.1}/tests/test_nvidia.py +0 -0
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gpuops
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: A small GPU operations CLI for shared lab servers.
|
|
5
5
|
Author: GPUOps contributors
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Requires-Python: >=3.9
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: rich>=13.7
|
|
9
10
|
|
|
10
11
|
# GPUOps
|
|
11
12
|
|
|
@@ -24,6 +25,8 @@ history.
|
|
|
24
25
|
- Per-user summary across all GPUs.
|
|
25
26
|
- Memory/running-time top view.
|
|
26
27
|
- Lightweight JSONL history sampling and GPU-hour aggregation.
|
|
28
|
+
- Rich terminal output: loading spinners, colored tables, memory bars, and progress bars.
|
|
29
|
+
- Interactive kill confirmation with `gpuops kill --interactive`.
|
|
27
30
|
- Safe-by-default process management: `kill` is dry-run unless `--yes` is passed.
|
|
28
31
|
|
|
29
32
|
## Install locally
|
|
@@ -52,6 +55,12 @@ gpuops kill --user alice
|
|
|
52
55
|
gpuops kill --gpu 3 --user alice
|
|
53
56
|
```
|
|
54
57
|
|
|
58
|
+
Preview and confirm in-place:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
gpuops kill --gpu 3 --user alice --interactive
|
|
62
|
+
```
|
|
63
|
+
|
|
55
64
|
To actually send signals:
|
|
56
65
|
|
|
57
66
|
```bash
|
|
@@ -59,6 +68,9 @@ gpuops kill --gpu 3 --user alice --yes
|
|
|
59
68
|
gpuops kill --user alice --force --yes
|
|
60
69
|
```
|
|
61
70
|
|
|
71
|
+
Use `--json` on any command when piping into scripts; JSON output skips the
|
|
72
|
+
loading spinners, prompts, and table styling.
|
|
73
|
+
|
|
62
74
|
## History
|
|
63
75
|
|
|
64
76
|
`gpuops status --record` appends one sample to:
|
|
@@ -15,6 +15,8 @@ history.
|
|
|
15
15
|
- Per-user summary across all GPUs.
|
|
16
16
|
- Memory/running-time top view.
|
|
17
17
|
- Lightweight JSONL history sampling and GPU-hour aggregation.
|
|
18
|
+
- Rich terminal output: loading spinners, colored tables, memory bars, and progress bars.
|
|
19
|
+
- Interactive kill confirmation with `gpuops kill --interactive`.
|
|
18
20
|
- Safe-by-default process management: `kill` is dry-run unless `--yes` is passed.
|
|
19
21
|
|
|
20
22
|
## Install locally
|
|
@@ -43,6 +45,12 @@ gpuops kill --user alice
|
|
|
43
45
|
gpuops kill --gpu 3 --user alice
|
|
44
46
|
```
|
|
45
47
|
|
|
48
|
+
Preview and confirm in-place:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
gpuops kill --gpu 3 --user alice --interactive
|
|
52
|
+
```
|
|
53
|
+
|
|
46
54
|
To actually send signals:
|
|
47
55
|
|
|
48
56
|
```bash
|
|
@@ -50,6 +58,9 @@ gpuops kill --gpu 3 --user alice --yes
|
|
|
50
58
|
gpuops kill --user alice --force --yes
|
|
51
59
|
```
|
|
52
60
|
|
|
61
|
+
Use `--json` on any command when piping into scripts; JSON output skips the
|
|
62
|
+
loading spinners, prompts, and table styling.
|
|
63
|
+
|
|
53
64
|
## History
|
|
54
65
|
|
|
55
66
|
`gpuops status --record` appends one sample to:
|
|
@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "gpuops"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.1"
|
|
8
8
|
description = "A small GPU operations CLI for shared lab servers."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
11
11
|
license = "MIT"
|
|
12
12
|
authors = [{ name = "GPUOps contributors" }]
|
|
13
|
-
dependencies = []
|
|
13
|
+
dependencies = ["rich>=13.7"]
|
|
14
14
|
|
|
15
15
|
[project.scripts]
|
|
16
16
|
gpuops = "gpuops.cli:main"
|
|
@@ -1,18 +1,30 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
|
-
import sys
|
|
5
4
|
from collections import defaultdict
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
from typing import List, Optional
|
|
8
7
|
|
|
9
8
|
from .actions import filter_processes, kill_processes, signal_for
|
|
10
9
|
from .alerts import evaluate_snapshot
|
|
11
|
-
from .formatting import
|
|
10
|
+
from .formatting import json_dump
|
|
12
11
|
from .history import DEFAULT_HISTORY_PATH, read_snapshots, record_snapshot, since_days, summarize_user_usage
|
|
12
|
+
from .models import Snapshot
|
|
13
13
|
from .nvidia import NvidiaSmiError, collect_snapshot
|
|
14
14
|
from .process_info import enrich_processes
|
|
15
|
-
from .
|
|
15
|
+
from .ui import (
|
|
16
|
+
confirm,
|
|
17
|
+
loading,
|
|
18
|
+
print_alerts,
|
|
19
|
+
print_empty,
|
|
20
|
+
print_error,
|
|
21
|
+
print_gpu_status,
|
|
22
|
+
print_history,
|
|
23
|
+
print_kill_results,
|
|
24
|
+
print_processes,
|
|
25
|
+
print_users,
|
|
26
|
+
progress,
|
|
27
|
+
)
|
|
16
28
|
|
|
17
29
|
|
|
18
30
|
def build_parser() -> argparse.ArgumentParser:
|
|
@@ -50,6 +62,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
50
62
|
kill.add_argument("--user")
|
|
51
63
|
kill.add_argument("--force", action="store_true", help="use SIGKILL instead of SIGTERM")
|
|
52
64
|
kill.add_argument("--yes", action="store_true", help="actually send the signal")
|
|
65
|
+
kill.add_argument("--interactive", "-i", action="store_true", help="preview and ask before sending signals")
|
|
53
66
|
|
|
54
67
|
history = subparsers.add_parser("history", help="summarize recorded GPU usage")
|
|
55
68
|
history.add_argument("--json", action="store_true", help="print machine-readable JSON")
|
|
@@ -59,9 +72,10 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
59
72
|
return parser
|
|
60
73
|
|
|
61
74
|
|
|
62
|
-
def _snapshot() -> Snapshot:
|
|
63
|
-
|
|
64
|
-
|
|
75
|
+
def _snapshot(show_loading: bool = True) -> Snapshot:
|
|
76
|
+
with loading("Collecting GPU telemetry", enabled=show_loading):
|
|
77
|
+
snap = collect_snapshot()
|
|
78
|
+
enrich_processes(snap.processes)
|
|
65
79
|
return snap
|
|
66
80
|
|
|
67
81
|
|
|
@@ -78,31 +92,35 @@ def _sort_gpus(gpus, key: str):
|
|
|
78
92
|
|
|
79
93
|
|
|
80
94
|
def cmd_status(args) -> int:
|
|
81
|
-
snap = _snapshot()
|
|
95
|
+
snap = _snapshot(show_loading=not args.json)
|
|
82
96
|
if args.record:
|
|
83
|
-
|
|
97
|
+
with loading("Recording history sample", enabled=not args.json):
|
|
98
|
+
record_snapshot(snap)
|
|
84
99
|
alerts = evaluate_snapshot(snap)
|
|
85
100
|
gpus = _sort_gpus(snap.gpus, args.sort)
|
|
86
101
|
if args.json:
|
|
87
102
|
print(json_dump({"snapshot": snap.to_dict(), "alerts": [alert.__dict__ for alert in alerts]}))
|
|
88
103
|
return 0
|
|
89
|
-
|
|
104
|
+
print_gpu_status(gpus, alerts)
|
|
90
105
|
return 0
|
|
91
106
|
|
|
92
107
|
|
|
93
108
|
def cmd_ps(args) -> int:
|
|
94
|
-
snap = _snapshot()
|
|
109
|
+
snap = _snapshot(show_loading=not args.json)
|
|
95
110
|
processes = filter_processes(snap.processes, gpu=args.gpu, user=args.user)
|
|
96
111
|
processes.sort(key=lambda proc: (proc.gpu_index if proc.gpu_index is not None else 9999, -proc.used_memory_mb))
|
|
97
112
|
if args.json:
|
|
98
113
|
print(json_dump([proc.__dict__ for proc in processes]))
|
|
99
114
|
return 0
|
|
100
|
-
|
|
115
|
+
if not processes:
|
|
116
|
+
print_empty("No matching GPU processes.")
|
|
117
|
+
return 0
|
|
118
|
+
print_processes(processes)
|
|
101
119
|
return 0
|
|
102
120
|
|
|
103
121
|
|
|
104
122
|
def cmd_users(args) -> int:
|
|
105
|
-
snap = _snapshot()
|
|
123
|
+
snap = _snapshot(show_loading=not args.json)
|
|
106
124
|
buckets = defaultdict(lambda: {"memory": 0, "processes": 0, "runtime": 0.0, "gpus": set()})
|
|
107
125
|
for proc in snap.processes:
|
|
108
126
|
user = proc.user or "unknown"
|
|
@@ -131,22 +149,15 @@ def cmd_users(args) -> int:
|
|
|
131
149
|
if args.json:
|
|
132
150
|
print(json_dump(summaries))
|
|
133
151
|
return 0
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
print(table(["User", "GPUs", "Processes", "Memory", "Total Running"], rows))
|
|
152
|
+
if not summaries:
|
|
153
|
+
print_empty("No active GPU users.")
|
|
154
|
+
return 0
|
|
155
|
+
print_users(summaries)
|
|
139
156
|
return 0
|
|
140
157
|
|
|
141
158
|
|
|
142
|
-
def _duration(seconds: float) -> str:
|
|
143
|
-
from .formatting import human_duration
|
|
144
|
-
|
|
145
|
-
return human_duration(seconds)
|
|
146
|
-
|
|
147
|
-
|
|
148
159
|
def cmd_top(args) -> int:
|
|
149
|
-
snap = _snapshot()
|
|
160
|
+
snap = _snapshot(show_loading=not args.json)
|
|
150
161
|
if args.by == "runtime":
|
|
151
162
|
processes = sorted(snap.processes, key=lambda proc: proc.running_seconds or 0, reverse=True)
|
|
152
163
|
else:
|
|
@@ -155,56 +166,69 @@ def cmd_top(args) -> int:
|
|
|
155
166
|
if args.json:
|
|
156
167
|
print(json_dump([proc.__dict__ for proc in processes]))
|
|
157
168
|
return 0
|
|
158
|
-
|
|
169
|
+
if not processes:
|
|
170
|
+
print_empty("No GPU processes to rank.")
|
|
171
|
+
return 0
|
|
172
|
+
print_processes(processes, title=f"GPU Process Top by {args.by.title()}")
|
|
159
173
|
return 0
|
|
160
174
|
|
|
161
175
|
|
|
162
176
|
def cmd_doctor(args) -> int:
|
|
163
|
-
snap = _snapshot()
|
|
177
|
+
snap = _snapshot(show_loading=not args.json)
|
|
164
178
|
alerts = evaluate_snapshot(snap, idle_util_percent=args.idle_util, occupied_memory_mb=args.occupied_mb)
|
|
165
179
|
if args.json:
|
|
166
180
|
print(json_dump([alert.__dict__ for alert in alerts]))
|
|
167
181
|
return 0
|
|
168
|
-
|
|
169
|
-
print("No suspicious GPU state found.")
|
|
170
|
-
return 0
|
|
171
|
-
rows = [[alert.level, alert.gpu_index, alert.message] for alert in alerts]
|
|
172
|
-
print(table(["Level", "GPU", "Message"], rows))
|
|
182
|
+
print_alerts(alerts)
|
|
173
183
|
return 1 if any(alert.level == "crit" for alert in alerts) else 0
|
|
174
184
|
|
|
175
185
|
|
|
176
186
|
def cmd_kill(args) -> int:
|
|
177
187
|
if args.gpu is None and args.user is None:
|
|
178
|
-
|
|
188
|
+
print_error("Refusing to select every GPU process. Pass --gpu, --user, or both.")
|
|
179
189
|
return 2
|
|
180
|
-
snap = _snapshot()
|
|
190
|
+
snap = _snapshot(show_loading=not args.json)
|
|
181
191
|
selected = filter_processes(snap.processes, gpu=args.gpu, user=args.user)
|
|
182
192
|
sig = signal_for(args.force)
|
|
183
|
-
|
|
184
|
-
|
|
193
|
+
dry_run = not args.yes
|
|
194
|
+
if args.interactive and selected and not args.yes and not args.json:
|
|
195
|
+
preview = kill_processes(selected, sig=sig, dry_run=True)
|
|
196
|
+
print_kill_results(preview)
|
|
197
|
+
dry_run = not confirm(f"Send {sig.name} to {len(selected)} selected process(es)?", default=False)
|
|
198
|
+
|
|
199
|
+
results = []
|
|
200
|
+
if selected and not dry_run and not args.json:
|
|
201
|
+
with progress("Sending signals", total=len(selected)) as bar:
|
|
202
|
+
task_id = bar.add_task("Sending signals", total=len(selected))
|
|
203
|
+
for proc in selected:
|
|
204
|
+
results.extend(kill_processes([proc], sig=sig, dry_run=False))
|
|
205
|
+
bar.advance(task_id)
|
|
206
|
+
else:
|
|
207
|
+
results = kill_processes(selected, sig=sig, dry_run=dry_run)
|
|
185
208
|
if args.json:
|
|
186
209
|
print(json_dump([result.__dict__ for result in results]))
|
|
187
210
|
return 0
|
|
188
|
-
if not
|
|
189
|
-
|
|
211
|
+
if not results:
|
|
212
|
+
print_empty("No matching GPU processes.")
|
|
190
213
|
return 0
|
|
191
|
-
|
|
192
|
-
if
|
|
193
|
-
|
|
214
|
+
print_kill_results(results)
|
|
215
|
+
if dry_run:
|
|
216
|
+
print_empty("Dry-run only. Re-run with --yes, or use --interactive to confirm in-place.")
|
|
194
217
|
return 0
|
|
195
218
|
|
|
196
219
|
|
|
197
220
|
def cmd_history(args) -> int:
|
|
198
|
-
|
|
199
|
-
|
|
221
|
+
with loading("Reading usage history", enabled=not args.json):
|
|
222
|
+
samples = read_snapshots(path=Path(args.path).expanduser(), since=since_days(args.days))
|
|
223
|
+
with loading("Summarizing GPU hours", enabled=not args.json):
|
|
224
|
+
usage = summarize_user_usage(samples)
|
|
200
225
|
if args.json:
|
|
201
226
|
print(json_dump([item.__dict__ for item in usage]))
|
|
202
227
|
return 0
|
|
203
228
|
if not usage:
|
|
204
|
-
|
|
229
|
+
print_empty("No history samples found. Run `gpuops status --record` periodically first.")
|
|
205
230
|
return 0
|
|
206
|
-
|
|
207
|
-
print(table(["User", "GPU Hours", "GB Hours", "Samples"], rows))
|
|
231
|
+
print_history(usage, args.days)
|
|
208
232
|
return 0
|
|
209
233
|
|
|
210
234
|
|
|
@@ -222,5 +246,5 @@ def main(argv: Optional[List[str]] = None) -> int:
|
|
|
222
246
|
"history": cmd_history,
|
|
223
247
|
}[args.command](args)
|
|
224
248
|
except NvidiaSmiError as exc:
|
|
225
|
-
|
|
249
|
+
print_error(str(exc))
|
|
226
250
|
return 127
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from contextlib import contextmanager
|
|
4
|
+
from typing import Iterable, Iterator, Sequence
|
|
5
|
+
|
|
6
|
+
from rich import box
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
from rich.panel import Panel
|
|
9
|
+
from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
|
10
|
+
from rich.prompt import Confirm
|
|
11
|
+
from rich.table import Table
|
|
12
|
+
from rich.text import Text
|
|
13
|
+
|
|
14
|
+
from .actions import KillResult
|
|
15
|
+
from .alerts import Alert
|
|
16
|
+
from .formatting import human_duration
|
|
17
|
+
from .history import UserUsage
|
|
18
|
+
from .models import GPU, GPUProcess
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
console = Console()
|
|
22
|
+
error_console = Console(stderr=True)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@contextmanager
|
|
26
|
+
def loading(message: str, enabled: bool = True) -> Iterator[None]:
|
|
27
|
+
if not enabled:
|
|
28
|
+
yield
|
|
29
|
+
return
|
|
30
|
+
with console.status(f"[bold cyan]{message}[/]", spinner="dots"):
|
|
31
|
+
yield
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def memory_bar(used_mb: int, total_mb: int, width: int = 18) -> Text:
|
|
35
|
+
ratio = 0 if total_mb <= 0 else min(1.0, used_mb / total_mb)
|
|
36
|
+
filled = round(ratio * width)
|
|
37
|
+
color = "green"
|
|
38
|
+
if ratio >= 0.9:
|
|
39
|
+
color = "red"
|
|
40
|
+
elif ratio >= 0.75:
|
|
41
|
+
color = "yellow"
|
|
42
|
+
text = Text()
|
|
43
|
+
text.append("█" * filled, style=color)
|
|
44
|
+
text.append("░" * (width - filled), style="dim")
|
|
45
|
+
text.append(f" {ratio * 100:4.0f}%", style=color)
|
|
46
|
+
return text
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def util_text(value: int) -> Text:
|
|
50
|
+
style = "green"
|
|
51
|
+
if value >= 90:
|
|
52
|
+
style = "red"
|
|
53
|
+
elif value >= 60:
|
|
54
|
+
style = "yellow"
|
|
55
|
+
elif value <= 5:
|
|
56
|
+
style = "dim"
|
|
57
|
+
return Text(f"{value}%", style=style)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def alert_badge(alerts: Sequence[str]) -> Text:
|
|
61
|
+
if not alerts:
|
|
62
|
+
return Text("healthy", style="green")
|
|
63
|
+
text = Text()
|
|
64
|
+
for index, alert in enumerate(alerts):
|
|
65
|
+
if index:
|
|
66
|
+
text.append("\n")
|
|
67
|
+
text.append(alert, style="bold yellow")
|
|
68
|
+
return text
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def print_gpu_status(gpus: Sequence[GPU], alerts: Sequence[Alert]) -> None:
|
|
72
|
+
alert_by_gpu: dict[int, list[str]] = {}
|
|
73
|
+
for alert in alerts:
|
|
74
|
+
alert_by_gpu.setdefault(alert.gpu_index, []).append(alert.message)
|
|
75
|
+
|
|
76
|
+
table = Table(title="GPUOps Status", box=box.ROUNDED, header_style="bold cyan", show_lines=False)
|
|
77
|
+
table.add_column("GPU", justify="right", style="bold", no_wrap=True)
|
|
78
|
+
table.add_column("Model", overflow="fold")
|
|
79
|
+
table.add_column("Memory", min_width=22)
|
|
80
|
+
table.add_column("Util", justify="right", no_wrap=True)
|
|
81
|
+
table.add_column("Temp", justify="right", no_wrap=True)
|
|
82
|
+
table.add_column("Power", justify="right", no_wrap=True)
|
|
83
|
+
table.add_column("State")
|
|
84
|
+
for gpu in gpus:
|
|
85
|
+
memory = Text()
|
|
86
|
+
memory.append(f"{gpu.memory_used_mb}/{gpu.memory_total_mb} MB\n", style="bold")
|
|
87
|
+
memory.append(memory_bar(gpu.memory_used_mb, gpu.memory_total_mb, width=10))
|
|
88
|
+
table.add_row(
|
|
89
|
+
str(gpu.index),
|
|
90
|
+
gpu.name,
|
|
91
|
+
memory,
|
|
92
|
+
util_text(gpu.utilization_gpu_percent),
|
|
93
|
+
"-" if gpu.temperature_c is None else f"{gpu.temperature_c}C",
|
|
94
|
+
"-" if gpu.power_w is None else f"{gpu.power_w:.0f}W",
|
|
95
|
+
alert_badge(alert_by_gpu.get(gpu.index, [])),
|
|
96
|
+
)
|
|
97
|
+
console.print(table)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def print_processes(processes: Sequence[GPUProcess], title: str = "GPU Processes") -> None:
|
|
101
|
+
table = Table(title=title, box=box.ROUNDED, header_style="bold cyan")
|
|
102
|
+
table.add_column("GPU", justify="right")
|
|
103
|
+
table.add_column("PID", justify="right", style="bold")
|
|
104
|
+
table.add_column("User")
|
|
105
|
+
table.add_column("Memory", justify="right")
|
|
106
|
+
table.add_column("Running", justify="right")
|
|
107
|
+
table.add_column("Command", overflow="fold")
|
|
108
|
+
table.add_column("CWD", overflow="fold", style="dim")
|
|
109
|
+
for proc in processes:
|
|
110
|
+
table.add_row(
|
|
111
|
+
"-" if proc.gpu_index is None else str(proc.gpu_index),
|
|
112
|
+
str(proc.pid),
|
|
113
|
+
proc.user or "-",
|
|
114
|
+
f"{proc.used_memory_mb} MB",
|
|
115
|
+
human_duration(proc.running_seconds),
|
|
116
|
+
proc.command or proc.process_name,
|
|
117
|
+
proc.cwd or "-",
|
|
118
|
+
)
|
|
119
|
+
console.print(table)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def print_users(summaries: Sequence[dict[str, object]]) -> None:
|
|
123
|
+
table = Table(title="GPU Usage by User", box=box.ROUNDED, header_style="bold cyan")
|
|
124
|
+
table.add_column("User", style="bold")
|
|
125
|
+
table.add_column("GPUs")
|
|
126
|
+
table.add_column("Processes", justify="right")
|
|
127
|
+
table.add_column("Memory", justify="right")
|
|
128
|
+
table.add_column("Total Running", justify="right")
|
|
129
|
+
for item in summaries:
|
|
130
|
+
table.add_row(
|
|
131
|
+
str(item["user"]),
|
|
132
|
+
str(item["gpus"]),
|
|
133
|
+
str(item["processes"]),
|
|
134
|
+
f"{item['memory_mb']} MB",
|
|
135
|
+
human_duration(float(item["runtime_seconds"])),
|
|
136
|
+
)
|
|
137
|
+
console.print(table)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def print_alerts(alerts: Sequence[Alert]) -> None:
|
|
141
|
+
if not alerts:
|
|
142
|
+
console.print(Panel("No suspicious GPU state found.", title="GPUOps Doctor", border_style="green"))
|
|
143
|
+
return
|
|
144
|
+
table = Table(title="GPUOps Doctor", box=box.ROUNDED, header_style="bold cyan")
|
|
145
|
+
table.add_column("Level")
|
|
146
|
+
table.add_column("GPU", justify="right")
|
|
147
|
+
table.add_column("Message")
|
|
148
|
+
for alert in alerts:
|
|
149
|
+
style = "bold red" if alert.level == "crit" else "yellow"
|
|
150
|
+
table.add_row(Text(alert.level, style=style), str(alert.gpu_index), Text(alert.message, style=style))
|
|
151
|
+
console.print(table)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def print_history(usage: Sequence[UserUsage], days: int) -> None:
|
|
155
|
+
table = Table(title=f"GPU Usage History ({days}d)", box=box.ROUNDED, header_style="bold cyan")
|
|
156
|
+
table.add_column("User", style="bold")
|
|
157
|
+
table.add_column("GPU Hours", justify="right")
|
|
158
|
+
table.add_column("GB Hours", justify="right")
|
|
159
|
+
table.add_column("Samples", justify="right")
|
|
160
|
+
for item in usage:
|
|
161
|
+
table.add_row(item.user, f"{item.gpu_hours:.2f}", f"{item.memory_gb_hours:.2f}", str(item.samples))
|
|
162
|
+
console.print(table)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def print_kill_results(results: Sequence[KillResult]) -> None:
|
|
166
|
+
is_plan = all(result.message == "dry-run" for result in results)
|
|
167
|
+
table = Table(title="Kill Plan" if is_plan else "Kill Results", box=box.ROUNDED, header_style="bold cyan")
|
|
168
|
+
table.add_column("GPU", justify="right")
|
|
169
|
+
table.add_column("PID", justify="right", style="bold")
|
|
170
|
+
table.add_column("User")
|
|
171
|
+
table.add_column("Signal")
|
|
172
|
+
table.add_column("Result")
|
|
173
|
+
for result in results:
|
|
174
|
+
style = "green" if result.killed else "yellow" if result.message == "dry-run" else "red"
|
|
175
|
+
table.add_row(
|
|
176
|
+
"-" if result.gpu_index is None else str(result.gpu_index),
|
|
177
|
+
str(result.pid),
|
|
178
|
+
result.user or "-",
|
|
179
|
+
result.signal_name,
|
|
180
|
+
Text(result.message, style=style),
|
|
181
|
+
)
|
|
182
|
+
console.print(table)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def confirm(message: str, default: bool = False) -> bool:
|
|
186
|
+
return Confirm.ask(message, default=default, console=console)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def progress(description: str, total: int) -> Progress:
|
|
190
|
+
return Progress(
|
|
191
|
+
SpinnerColumn(),
|
|
192
|
+
TextColumn("[bold cyan]{task.description}"),
|
|
193
|
+
BarColumn(),
|
|
194
|
+
TextColumn("{task.completed}/{task.total}"),
|
|
195
|
+
TimeElapsedColumn(),
|
|
196
|
+
console=console,
|
|
197
|
+
transient=False,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def print_empty(message: str) -> None:
|
|
202
|
+
console.print(Panel(message, border_style="dim"))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def print_error(message: str) -> None:
|
|
206
|
+
error_console.print(f"[bold red]error:[/] {message}")
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gpuops
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: A small GPU operations CLI for shared lab servers.
|
|
5
5
|
Author: GPUOps contributors
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Requires-Python: >=3.9
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: rich>=13.7
|
|
9
10
|
|
|
10
11
|
# GPUOps
|
|
11
12
|
|
|
@@ -24,6 +25,8 @@ history.
|
|
|
24
25
|
- Per-user summary across all GPUs.
|
|
25
26
|
- Memory/running-time top view.
|
|
26
27
|
- Lightweight JSONL history sampling and GPU-hour aggregation.
|
|
28
|
+
- Rich terminal output: loading spinners, colored tables, memory bars, and progress bars.
|
|
29
|
+
- Interactive kill confirmation with `gpuops kill --interactive`.
|
|
27
30
|
- Safe-by-default process management: `kill` is dry-run unless `--yes` is passed.
|
|
28
31
|
|
|
29
32
|
## Install locally
|
|
@@ -52,6 +55,12 @@ gpuops kill --user alice
|
|
|
52
55
|
gpuops kill --gpu 3 --user alice
|
|
53
56
|
```
|
|
54
57
|
|
|
58
|
+
Preview and confirm in-place:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
gpuops kill --gpu 3 --user alice --interactive
|
|
62
|
+
```
|
|
63
|
+
|
|
55
64
|
To actually send signals:
|
|
56
65
|
|
|
57
66
|
```bash
|
|
@@ -59,6 +68,9 @@ gpuops kill --gpu 3 --user alice --yes
|
|
|
59
68
|
gpuops kill --user alice --force --yes
|
|
60
69
|
```
|
|
61
70
|
|
|
71
|
+
Use `--json` on any command when piping into scripts; JSON output skips the
|
|
72
|
+
loading spinners, prompts, and table styling.
|
|
73
|
+
|
|
62
74
|
## History
|
|
63
75
|
|
|
64
76
|
`gpuops status --record` appends one sample to:
|
|
@@ -10,10 +10,12 @@ src/gpuops/history.py
|
|
|
10
10
|
src/gpuops/models.py
|
|
11
11
|
src/gpuops/nvidia.py
|
|
12
12
|
src/gpuops/process_info.py
|
|
13
|
+
src/gpuops/ui.py
|
|
13
14
|
src/gpuops.egg-info/PKG-INFO
|
|
14
15
|
src/gpuops.egg-info/SOURCES.txt
|
|
15
16
|
src/gpuops.egg-info/dependency_links.txt
|
|
16
17
|
src/gpuops.egg-info/entry_points.txt
|
|
18
|
+
src/gpuops.egg-info/requires.txt
|
|
17
19
|
src/gpuops.egg-info/top_level.txt
|
|
18
20
|
tests/test_alerts_history_actions.py
|
|
19
21
|
tests/test_nvidia.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
rich>=13.7
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|