gpu-dev 0.7.5__tar.gz → 0.7.6__tar.gz

{gpu_dev-0.7.5 → gpu_dev-0.7.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.7.5
+Version: 0.7.6
 Summary: CLI + Python SDK for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.7.5 → gpu_dev-0.7.6}/cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py

@@ -1523,12 +1523,19 @@ def reserve(
 @click.option("--gpu-type", default="b200", show_default=True, help="GPU type for the repro box.")
 @click.option("--gpus", type=int, default=1, show_default=True)
 @click.option("--hours", type=float, default=3.0, show_default=True,
-              help="Lifetime ceiling; the box auto-cancels when the test exits unless --keep.")
+              help="Lifetime ceiling for the box.")
+@click.option("--no-connect", is_flag=True, default=False,
+              help="CI mode: run the test, auto-cancel, exit code = test result. Default (on a TTY) drops you into the box to iterate.")
 @click.option("--keep", is_flag=True, default=False,
-              help="Keep the reservation after the test exits (default: auto-cancel).")
+              help="Never cancel the box (skip the cancel prompt / auto-cancel).")
 @click.pass_context
-def repro(ctx, ref, test_args, gpu_type, gpus, hours, keep):
-    """Reserve a GPU, check out a PR/commit, run a test, then auto-cancel.
+def repro(ctx, ref, test_args, gpu_type, gpus, hours, no_connect, keep):
+    """Reserve a GPU, check out a PR/commit, run a test, then drop you into the box.
+
+    By default (in a terminal) repro runs the test and then **connects you into the
+    box** at ~/pytorch — the ref is checked out, so you can fix and re-run. The box
+    stays alive until you cancel it (you're prompted on exit). Use --no-connect for
+    CI/scripts (run the test, auto-cancel, process exit code = the test result).
 
     REF: pr/<N>, #<N>, a bare PR number, a branch, or a commit sha. PRs use
     pull/<N>/merge (what CI tests), falling back to /head.
@@ -1539,6 +1546,7 @@ def repro(ctx, ref, test_args, gpu_type, gpus, hours, keep):
     """
     import shlex
     import subprocess
+    import sys
     config = load_config()
     reservation_mgr = ReservationManager(config)
     try:
@@ -1602,21 +1610,55 @@ def repro(ctx, ref, test_args, gpu_type, gpus, hours, keep):
     if "StrictHostKeyChecking" not in ssh_cmd:
         ssh_cmd = ssh_cmd.replace("ssh ", "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR ", 1)
     rprint(f"[dim]→ {ssh_cmd}[/dim]\n")
+    rid8 = str(rid)[:8]
     rc = 1
     try:
         rc = subprocess.run(f"{ssh_cmd} {shlex.quote(remote)}", shell=True).returncode
     except KeyboardInterrupt:
-        rprint("\n[yellow]interrupted[/yellow]")
-    finally:
+        rprint("\n[yellow]interrupted[/yellow]"); rc = 130
+
+    verdict = "[green]✓ test passed[/green]" if rc == 0 else f"[red]✗ test failed (exit {rc})[/red]"
+
+    # Default (TTY): drop into the box so you can fix and re-run. --no-connect is the
+    # CI path: auto-cancel and exit with the test's code.
+    connect = (not no_connect) and sys.stdout.isatty()
+    if connect:
+        rprint(f"\n{verdict} — dropping you into the box at ~/pytorch ({ref} checked out).")
+        rprint(f"[dim]  re-run:  python {testcmd}[/dim]")
+        rprint(f"[dim]  finish:  gpu-dev cancel  (from inside)  •  or exit this shell[/dim]\n")
+        shell_cmd = f"{ssh_cmd} -t {shlex.quote('cd /home/dev/pytorch 2>/dev/null; exec ${SHELL:-bash} -l')}"
+        try:
+            subprocess.run(shell_cmd, shell=True)
+        except KeyboardInterrupt:
+            pass
         if keep:
-            rprint(f"[cyan]📌 kept {str(rid)[:8]} — gpu-dev connect {str(rid)[:8]} • gpu-dev cancel {str(rid)[:8]}[/cyan]")
-        else:
+            rprint(f"[cyan]📌 left {rid8} running — connect: gpu-dev connect {rid8} • cancel: gpu-dev cancel {rid8}[/cyan]")
+            return
+        try:
+            drop = click.confirm(f"Cancel repro box {rid8}?", default=True)
+        except (KeyboardInterrupt, EOFError, click.Abort):
+            drop = False
+        if drop:
             try:
                 reservation_mgr.cancel_reservation(rid, user_info["user_id"])
-                rprint(f"[green]🧹 cancelled repro box {str(rid)[:8]}[/green]")
+                rprint(f"[green]🧹 cancelled {rid8}[/green]")
             except Exception as e:
-                rprint(f"[yellow]auto-cancel failed for {str(rid)[:8]}: {e}[/yellow]")
-    rprint(f"\n[bold]repro exit code: {rc}[/bold]")
+                rprint(f"[yellow]cancel failed for {rid8}: {e}[/yellow]")
+        else:
+            rprint(f"[cyan]📌 left {rid8} running — connect: gpu-dev connect {rid8} • cancel: gpu-dev cancel {rid8}[/cyan]")
+        return
+
+    # --no-connect / non-TTY: auto-cancel unless --keep, exit code = test result.
+    if keep:
+        rprint(f"[cyan]📌 kept {rid8} — gpu-dev connect {rid8} • gpu-dev cancel {rid8}[/cyan]")
+    else:
+        try:
+            reservation_mgr.cancel_reservation(rid, user_info["user_id"])
+            rprint(f"[green]🧹 cancelled repro box {rid8}[/green]")
+        except Exception as e:
+            rprint(f"[yellow]auto-cancel failed for {rid8}: {e}[/yellow]")
+    rprint(f"\n[bold]repro exit code: {rc}[/bold] ({verdict})")
+    sys.exit(rc)
 
 
 _SUBMIT_GPU_TYPES = ["b300", "b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g", "h200", "h100",

{gpu_dev-0.7.5 → gpu_dev-0.7.6}/docs/SDK_REPRO.md

@@ -2,7 +2,10 @@
 
 Reserve a **warm** GPU box in ~1s, run code on it, and auto-clean up — from Python
 or one CLI command. Backed by a pool of pre-booted pods with **PyTorch prebuilt**
-(viable/strict), so `import torch` works instantly with no build.
+(viable/strict), so `import torch` works instantly with no build. And when you *do*
+have to compile — your ref moved past viable/strict, or you touch C++ — a shared
+compiler cache (ccache) makes it an **incremental, not a cold, build** (see
+[Builds are cached](#builds-are-cached-shared-ccache)).
 
 > Requires `gpu-dev` ≥ 0.7.1 (CLI **and** SDK in one package): `pip install --upgrade gpu-dev`
 
@@ -32,14 +35,18 @@ with client.reserve(gpu_type="b200", gpu_count=1, hours=1) as sb:
 PyTorch is pre-staged at `~/pytorch` (importable). To reproduce a failure, point at
 the **PR or commit** and run the test.
 
-**One CLI command** (reserve → checkout → run → auto-cancel):
+**One CLI command** (reserve → checkout → run → **drop you into the box to fix**):
 ```bash
 gpu-dev repro pr/185264 test/inductor/test_flex_attention.py TestFlexAttentionCUDA.test_large_kv_int64_pointer_math_cuda
 ```
 - `REF`: `pr/<N>`, `#<N>`, a bare PR number, a branch, or a commit sha.
 - PRs use **`pull/<N>/merge`** (what CI actually tests — the PR merged onto current
   trunk), falling back to `/head`. Use this, not the raw branch.
-- `--keep` to inspect afterward instead of auto-cancelling.
+- By default (in a terminal) repro runs the test, prints the verdict, then **lands
+  you in the box** at `~/pytorch` with the ref checked out so you can fix and re-run;
+  it stays alive until you cancel (prompted on exit).
+- `--no-connect` = CI mode: run, auto-cancel, process exit code = the test result.
+- `--keep` never cancels (no prompt). `--gpu-type` / `--gpus` / `--hours` to size it.
 
 **From the SDK:**
 ```python
@@ -62,12 +69,48 @@ pip install -e . --no-build-isolation
 ```
 Python-only changes need no rebuild — `PYTHONPATH=~/pytorch` already resolves.
 
+## Builds are cached (shared ccache)
+
+Two layers of caching mean you almost never pay for a cold, from-scratch build —
+including the full C++/CUDA compile (gcc/nvcc):
+
+1. **Prebuilt tree.** Every box gets PyTorch already built at viable/strict and
+   staged at `~/pytorch`, so `import torch` works with **zero build**.
+2. **Shared compiler cache (ccache).** `CCACHE_DIR=/ccache_shared` is an EFS volume
+   mounted in **every** dev pod *and* the dedicated build node, so all the C++/CUDA
+   object compiles are cached and **shared across users and the build node**. When
+   you check out a ref past viable/strict — or edit C++ — the rebuild reuses those
+   cached objects (and the warm `build/` for ninja) instead of recompiling from
+   scratch. So even a "full" `pip install -e .` is a warm build, not a cold one.
+
+Measured (m7i build node, 128 jobs, CUDA 13.2):
+
+| scenario | time |
+|---|---|
+| `import torch` (prebuilt, no build) | ~0s |
+| incremental (1 kernel changed + relink) | ~40s |
+| ninja no-op (nothing changed) | ~20s |
+| from-scratch `build/` with warm ccache (~86% hit) | ~21 min |
+
+(A true cold build from an empty ccache is far longer.) The cache stays warm on its
+own: an hourly build-node job compiles each viable/strict bump into `/ccache_shared`,
+so the objects you need are usually already there by the time you build — and your
+own compiles populate it for the next person too.
+
 ## Gotchas
 - **`/merge` vs `/head`**: `/head` is the PR author's raw branch and often lacks
   trunk-added tests; `/merge` is what CI ran. `gpu-dev repro` / `--ref` use `/merge`.
 - **The prebuilt is viable/strict.** If your ref moved past it and a test needs new
-  C++, do the one incremental `pip install -e . --no-build-isolation`.
+  C++, do the one incremental `pip install -e . --no-build-isolation` — it's fast
+  (warm shared ccache), not a cold build. See [Builds are cached](#builds-are-cached-shared-ccache).
 - **Ephemeral by design.** Repro boxes have no persistent disk; bring code via
   `--ref`, `sb.upload`, or git.
+- **Reproducing a reverted PR / an OOM.** `pr/N` uses `/merge` = the PR re-applied
+  onto *current* trunk — so if the PR was reverted, `/merge` effectively un-reverts
+  it and you test the **fixed** tree (it'll pass). To repro the failing trunk state,
+  check out the **exact land commit** instead (`gpu-dev repro <sha> …`). And match the
+  CI runner's GPU: an **OOM** only reproduces on a GPU as small as the runner's — the
+  default `b200` has far more memory, so a memory-bound failure won't show there
+  (`--gpu-type h100`/`a100`/… to match).
 
 See also: `sdk/python/README.md` and `sdk/python/examples/`.

{gpu_dev-0.7.5 → gpu_dev-0.7.6}/gpu_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.7.5
+Version: 0.7.6
 Summary: CLI + Python SDK for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.7.5 → gpu_dev-0.7.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gpu-dev"
-version = "0.7.5"
+version = "0.7.6"
 description = "CLI + Python SDK for PyTorch GPU developer server reservations"
 authors = [{name = "PyTorch Team"}]
 readme = "cli-tools/gpu-dev-cli/README.md"

{gpu_dev-0.7.5 → gpu_dev-0.7.6}/sdk/python/src/gpu_dev/__init__.py

@@ -63,4 +63,4 @@ try:
 
     __version__ = _pkg_version("gpu-dev")
 except Exception:
-    __version__ = "0.7.5"
+    __version__ = "0.7.6"

{gpu_dev-0.7.5 → gpu_dev-0.7.6}/terraform-gpu-devservers/lambda/reservation_processor/index.py

@@ -5283,10 +5283,11 @@ EOF_PROFILE
 
 # User identification
 export GPU_DEV_USER_ID="{user_id or 'dev'}"
-# Reservation id — from the pod hostname (gpu-dev-<id>). Warm claims overwrite this
-# in place with the full id; cold pods carry the 8-char prefix (cancellation resolves
-# by prefix). Lets `gpu-dev cancel` with no args inside the pod stop this reservation.
-export GPU_DEV_RESERVATION_ID="$(hostname | sed -e 's/^gpu-dev-//')"
+# Reservation id from the pod hostname; warm claims overwrite it with the full id,
+# cold pods keep the 8-char prefix. Used by gpu-dev cancel (no args) inside the pod.
+# NOTE: escape the dollar so this is evaluated when the shell sources the file, NOT
+# command-substituted while this (unquoted) heredoc is written at pod startup.
+export GPU_DEV_RESERVATION_ID="\$(hostname | sed -e 's/^gpu-dev-//')"
 
 # Multinode peer info — inlined from container env at pod startup. sshd strips
 # container env vars from login shells, so we materialize the values into rc files.
@@ -5353,10 +5354,11 @@ EOF_BASHRC_EXT
 
 # User identification
 export GPU_DEV_USER_ID="{user_id or 'dev'}"
-# Reservation id — from the pod hostname (gpu-dev-<id>). Warm claims overwrite this
-# in place with the full id; cold pods carry the 8-char prefix (cancellation resolves
-# by prefix). Lets `gpu-dev cancel` with no args inside the pod stop this reservation.
-export GPU_DEV_RESERVATION_ID="$(hostname | sed -e 's/^gpu-dev-//')"
+# Reservation id from the pod hostname; warm claims overwrite it with the full id,
+# cold pods keep the 8-char prefix. Used by gpu-dev cancel (no args) inside the pod.
+# NOTE: escape the dollar so this is evaluated when the shell sources the file, NOT
+# command-substituted while this (unquoted) heredoc is written at pod startup.
+export GPU_DEV_RESERVATION_ID="\$(hostname | sed -e 's/^gpu-dev-//')"
 
 # Multinode peer info — inlined from container env at pod startup. sshd strips
 # container env vars from login shells, so we materialize the values into rc files.