sgpu 0.6.0__py3-none-any.whl

sgpu/__init__.py

@@ -0,0 +1,3 @@
+"""sgpu — simple GPU monitor client for the SGVR lab MLXP cluster."""
+
+__version__ = "0.6.0"

sgpu/__main__.py

@@ -0,0 +1,4 @@
+from sgpu.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sgpu/cli.py

@@ -0,0 +1,176 @@
+"""sgpu command-line client.
+
+A thin cross-platform wrapper: all rendering happens inside the monitor pod,
+this client only shells out to kubectl (fetch server-rendered text, or hand
+the terminal to the in-pod TUI via `kubectl exec -it`).
+"""
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+from sgpu import __version__
+
+DEFAULT_NAMESPACE = os.environ.get("SGPU_NAMESPACE", "p-sgvr-node-02")
+DEFAULT_POD = os.environ.get("SGPU_POD", "sangmin-gpu-monitor")
+DEPLOY_URL = ("https://raw.githubusercontent.com/alex6095/sgpu/main/"
+              "k8s/gpu-monitor.yaml")
+
+COMMANDS = ("top", "once", "watch", "apps", "stats", "nvitop", "pods",
+            "smi", "gpustat", "json", "health", "version")
+
+EPILOG = """commands:
+  (none) | top     interactive TUI (scroll, sort, owner filter)
+  once             one-shot dashboard
+  watch [sec]      simple refresh loop (for dumb terminals)
+  apps             GPU process table with pod owners
+  stats [days]     per-owner usage report (default 7 days)
+  nvitop           raw nvitop TUI
+  pods | smi | gpustat | json | health | version
+"""
+
+
+def _enable_ansi():
+    if os.name == "nt":
+        os.system("")  # nudges conhost/Windows Terminal into VT mode
+    try:
+        sys.stdout.reconfigure(encoding="utf-8")
+        sys.stderr.reconfigure(encoding="utf-8")
+    except (AttributeError, OSError):
+        pass
+
+
+def _hint_on_failure(args, stderr_text):
+    print("sgpu: cannot reach monitor pod '%s' in namespace '%s'"
+          % (args.pod, args.namespace), file=sys.stderr)
+    tail = [line for line in stderr_text.strip().splitlines()][-3:]
+    for line in tail:
+        print(line, file=sys.stderr)
+    lowered = stderr_text.lower()
+    if "notfound" in lowered or "not found" in lowered:
+        print("hint: the monitor pod is not running. Deploy it with:",
+              file=sys.stderr)
+        print("  kubectl apply -f " + DEPLOY_URL, file=sys.stderr)
+    elif "unauthorized" in lowered or "forbidden" in lowered \
+            or "credentials" in lowered:
+        print("hint: check your access: kubectl auth can-i get pods -n %s"
+              % args.namespace, file=sys.stderr)
+
+
+def _fetch(args, path):
+    result = subprocess.run(
+        ["kubectl", "exec", "-n", args.namespace, args.pod, "--",
+         "curl", "-fsS", "http://127.0.0.1:8080" + path],
+        capture_output=True, text=True, encoding="utf-8", errors="replace")
+    if result.returncode != 0:
+        _hint_on_failure(args, result.stderr or result.stdout)
+        return None
+    return result.stdout
+
+
+def _color_param(args):
+    use = (not args.no_color) and sys.stdout.isatty()
+    return "color=1" if use else "color=0"
+
+
+def _cols_param():
+    cols = shutil.get_terminal_size(fallback=(120, 40)).columns
+    return "cols=%d" % max(40, cols)
+
+
+def _print(text):
+    if text is not None:
+        sys.stdout.write(text)
+        if not text.endswith("\n"):
+            sys.stdout.write("\n")
+
+
+def _interactive(args, pod_command):
+    if not (sys.stdout.isatty() and sys.stdin.isatty()):
+        _print(_fetch(args, "/table?color=0&" + _cols_param()))
+        return 0
+    return subprocess.call(
+        ["kubectl", "exec", "-it", "-n", args.namespace, args.pod, "--"]
+        + pod_command)
+
+
+def _watch(args):
+    seconds = args.number if args.number else args.refresh
+    esc = "\x1b"
+    sys.stdout.write(esc + "[?1049h" + esc + "[?25l")
+    try:
+        while True:
+            frame = _fetch(args, "/table?%s&%s"
+                           % (_color_param(args), _cols_param()))
+            if frame is None:
+                frame = "sgpu: fetch failed; retrying..."
+            sys.stdout.write(esc + "[H" + frame + "\n" + esc + "[0J")
+            sys.stdout.flush()
+            time.sleep(max(1, seconds))
+    except KeyboardInterrupt:
+        return 0
+    finally:
+        sys.stdout.write(esc + "[?25h" + esc + "[?1049l")
+        sys.stdout.flush()
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        prog="sgpu",
+        description="Simple GPU monitor for the SGVR lab MLXP cluster.",
+        epilog=EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("command", nargs="?", default="top",
+                        choices=COMMANDS, metavar="command")
+    parser.add_argument("number", nargs="?", type=int, default=0,
+                        metavar="N",
+                        help="days for stats, seconds for watch")
+    parser.add_argument("-n", "--namespace", default=DEFAULT_NAMESPACE)
+    parser.add_argument("--pod", default=DEFAULT_POD)
+    parser.add_argument("-r", "--refresh", type=int, default=2,
+                        help="TUI/watch refresh interval (default 2)")
+    parser.add_argument("--no-color", action="store_true")
+    parser.add_argument("-V", "--version", action="version",
+                        version="sgpu " + __version__)
+    args = parser.parse_args(argv)
+
+    if shutil.which("kubectl") is None:
+        print("sgpu: kubectl not found on PATH", file=sys.stderr)
+        print("hint: install kubectl and configure the MLXP kubeconfig "
+              "(see the sgpu README)", file=sys.stderr)
+        return 127
+
+    _enable_ansi()
+    query = "%s&%s" % (_color_param(args), _cols_param())
+
+    if args.command == "top":
+        return _interactive(args, ["python3", "/opt/gpu-monitor/tui.py",
+                                   str(args.refresh)])
+    if args.command == "nvitop":
+        return _interactive(args, ["nvitop"])
+    if args.command == "watch":
+        return _watch(args)
+
+    paths = {
+        "once": "/table?" + query,
+        "apps": "/apps?" + query,
+        "stats": "/stats?days=%d&%s" % (args.number or 7, query),
+        "pods": "/pods",
+        "smi": "/smi",
+        "gpustat": "/gpustat",
+        "json": "/json",
+        "health": "/health",
+        "version": "/version",
+    }
+    text = _fetch(args, paths[args.command])
+    if text is None:
+        return 1
+    _print(text)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

sgpu-0.6.0.dist-info/METADATA

@@ -0,0 +1,128 @@
+Metadata-Version: 2.4
+Name: sgpu
+Version: 0.6.0
+Summary: Simple GPU monitor for the SGVR H200 lab (MLXP) - kubectl-native dashboards, in-pod TUI, per-user usage stats
+Author: alex6095
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/alex6095/sgpu
+Project-URL: Repository, https://github.com/alex6095/sgpu
+Keywords: gpu,kubernetes,monitoring,nvml,nvitop,h200
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: System :: Monitoring
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# SGPU
+
+**SGVR GPU** / **S**imple **GPU** monitor for the lab's MLXP H200 nodes —
+check the GPUs before you launch a pod.
+
+- Every GPU process is attributed to its **pod and owner** (not just a PID).
+- **In-pod TUI** (`kubectl exec -it`) — smooth like nvitop, scroll/sort/filter.
+- **Usage stats 24/7**: per-owner GPU-hours, awards, GitHub-style activity
+  calendar, idle-allocation warnings (KST).
+- Shared **storage (pv-01/pv-02) usage** at a glance.
+- Monitor pod is read-only, always-on, and requests **no GPU**.
+
+## Install
+
+```bash
+uvx sgpu            # run without installing (uv)
+pipx install sgpu   # or pipx
+pip install sgpu    # or plain pip (WSL/Ubuntu: add --user --break-system-packages)
+```
+
+Needs `kubectl` configured for the MLXP namespace
+([kubectl setup](#kubectl-setup-linuxwsl)).
+
+## Use
+
+```text
+sgpu               interactive TUI      sgpu stats [days]  usage report + awards
+sgpu once          one-shot dashboard   sgpu apps          processes + owners
+sgpu watch [sec]   dumb-terminal loop   sgpu nvitop        raw nvitop
+sgpu pods|smi|gpustat|json|health|version|--help
+```
+
+TUI keys: `j/k` scroll · `Tab` pane · `s` sort · `o` owner filter · `p` pause · `q` quit.
+Options: `-n` namespace, `--pod`, `-r` refresh, `--no-color`. Env: `SGPU_NAMESPACE`, `SGPU_POD`.
+
+### Zero-install (kubectl only)
+
+```bash
+kubectl exec -it -n p-sgvr-node-02 sangmin-gpu-monitor -- python3 /opt/gpu-monitor/tui.py
+kubectl exec -n p-sgvr-node-02 sangmin-gpu-monitor -- curl -fsS http://127.0.0.1:8080/table
+```
+
+Endpoints on `:8080`: `/table /apps /json /stats /pods /smi /topo /gpustat
+/health /version /stats/files /stats/raw?date=` (`?color=1&cols=N&ascii=1`).
+
+## Stats
+
+Sampled every 15 s around the clock into raw JSONL (full fidelity — future
+tools can recompute anything), gzipped + rolled up daily, stored on the
+shared volume at `pv-01/sangmin/sgpu`. Retention 365 d, capped at 2 GB.
+`sgpu stats 30` shows the leaderboard, awards, daily activity calendar and
+KST hour heatmap. Raw export: `/stats/raw?date=YYYYMMDD`.
+
+> The monitor pod must stay running for stats to accumulate — it is designed
+> to (tini init, `restartPolicy: Always`, no GPU held).
+
+## Deploy / operate
+
+Image push works **from anywhere** via the registry's public endpoint
+(`sgvr-registry.kr.ncr.ntruss.com`); the cluster pulls via the private
+endpoint (`vnxb4cz3.kr.private-ncr.ntruss.com`, in-cluster only — preferred
+per the MLXP guide). API key: NCP console → Access Management.
+
+```bash
+docker login sgvr-registry.kr.ncr.ntruss.com
+docker build -f docker/Dockerfile.gpu-monitor -t sgvr-registry.kr.ncr.ntruss.com/sangmin/gpu-monitor:TAG .
+docker push sgvr-registry.kr.ncr.ntruss.com/sangmin/gpu-monitor:TAG
+
+kubectl delete pod sangmin-gpu-monitor -n p-sgvr-node-02 --ignore-not-found
+kubectl apply -f k8s/gpu-monitor.yaml   # pods are immutable: delete + apply
+kubectl wait --for=condition=Ready pod/sangmin-gpu-monitor -n p-sgvr-node-02 --timeout=180s
+```
+
+Optional (enables the pod-allocation view + idle stats; kubelet syncs it in
+within a minute, no restart):
+
+```bash
+kubectl -n p-sgvr-node-02 create secret generic sgpu-kubeconfig --from-file=config=$HOME/.kube/config
+```
+
+> Anyone with exec access to the monitor pod can read that token — fine
+> inside a trusting lab namespace; use your least-privileged kubeconfig.
+
+## kubectl setup (Linux/WSL)
+
+```bash
+mkdir -p ~/.local/bin ~/.kube
+V=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
+curl -fsSL -o ~/.local/bin/kubectl "https://dl.k8s.io/release/${V}/bin/linux/amd64/kubectl" && chmod +x ~/.local/bin/kubectl
+cp /path/to/sgvr-node-02-kubeconfig.yaml ~/.kube/config && chmod 600 ~/.kube/config
+kubectl get pods -n p-sgvr-node-02   # connectivity test
+```
+
+## Development
+
+```bash
+SGPU_MOCK=1 python3 tools/gpu-monitor/server.py   # full pipeline, no GPU needed
+SGPU_MOCK=1 python3 tools/gpu-monitor/tui.py
+python3 -m unittest discover -s tests
+```
+
+How it works: `sgpu` (thin Python client) → `kubectl exec` → monitor pod
+(privileged, hostPID) where server.py renders everything; process→pod
+attribution reads `/proc/<pid>/environ` (`HOSTNAME` = pod name), owner =
+pod-name prefix. Known limits: pods overriding `spec.hostname` and MPS show
+as `?`.
+
+Troubleshooting: broken terminal after a dropped TUI → `reset` · frozen TUI
+→ rerun `sgpu` · garbled bars → Windows Terminal or `--no-color`.

sgpu-0.6.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+sgpu/__init__.py,sha256=02Y_EpE47WPTsR4dFog4XGjX8FXOmO-dZ5S2B6NJr1g,95
+sgpu/__main__.py,sha256=bjHD0BX4-wzy-m8I1CUWB-liI4AI9GgpfphxdUNvm5Y,83
+sgpu/cli.py,sha256=COy4aAjHc5HNVO0nZ5XxNQ1rmVG6kTFMXl9JeCU-XUo,6038
+sgpu-0.6.0.dist-info/licenses/LICENSE,sha256=SSj-3-ZHkxfJbyUqBfJKSXPGh2qjSybJ0YOzM5Tl7Bw,1065
+sgpu-0.6.0.dist-info/METADATA,sha256=6Ia-pvo6M2Onzt1NQudgeELZTcgR9mgdOhTB0urMUU0,5082
+sgpu-0.6.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+sgpu-0.6.0.dist-info/entry_points.txt,sha256=8XnjqiFHDVElaiefyPH22fOCtaJMKDq6fq9NuwjUnTU,39
+sgpu-0.6.0.dist-info/top_level.txt,sha256=8KJ5Yrh8tD377mC48ka56WkrwUqUb5pg9EXv24J4XZw,5
+sgpu-0.6.0.dist-info/RECORD,,

sgpu-0.6.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sgpu-0.6.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sgpu = sgpu.cli:main

sgpu-0.6.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 alex6095
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sgpu-0.6.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sgpu