gpu-dev 0.7.11__tar.gz → 0.7.13__tar.gz

{gpu_dev-0.7.11 → gpu_dev-0.7.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.7.11
+Version: 0.7.13
 Summary: CLI + Python SDK for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.7.11 → gpu_dev-0.7.13}/cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py

@@ -1521,19 +1521,23 @@ def reserve(
 
 
 @main.command(context_settings={"ignore_unknown_options": True})
-@click.argument("ref")
-@click.argument("test_args", nargs=-1, required=True)
-@click.option("--gpu-type", default="b200", show_default=True, help="GPU type for the repro box.")
+@click.argument("ref", required=False)
+@click.argument("test_args", nargs=-1, required=False)
+@click.option("--lint", is_flag=True, default=False,
+              help="Run a PyTorch lint job (lintrunner) on a CPU box instead of a python test. "
+                   "Defaults to --gpu-type cpu-x86 and skips the torch build. Extra args go to "
+                   "lintrunner (default: --merge-base-with origin/main, i.e. the PR diff like CI).")
+@click.option("--gpu-type", default=None, help="GPU type for the repro box (default: b200; cpu-x86 with --lint).")
 @click.option("--gpus", type=int, default=1, show_default=True)
 @click.option("--hours", type=float, default=3.0, show_default=True,
               help="Lifetime ceiling for the box.")
 @click.option("--no-connect", is_flag=True, default=False,
-              help="CI mode: run the test, auto-cancel, exit code = test result. Default (on a TTY) drops you into the box to iterate.")
+              help="CI mode: run the test/lint, auto-cancel, exit code = result. Default (on a TTY) drops you into the box to iterate.")
 @click.option("--keep", is_flag=True, default=False,
               help="Never cancel the box (skip the cancel prompt / auto-cancel).")
 @click.pass_context
-def repro(ctx, ref, test_args, gpu_type, gpus, hours, no_connect, keep):
-    """Reserve a GPU, check out a PR/commit, run a test, then drop you into the box.
+def repro(ctx, ref, test_args, lint, gpu_type, gpus, hours, no_connect, keep):
+    """Reserve a box, check out a PR/commit, run a test (or lint), then drop you in.
 
     By default (in a terminal) repro runs the test and then **connects you into the
     box** at ~/pytorch — the ref is checked out, so you can fix and re-run. The box
@@ -1546,10 +1550,30 @@ def repro(ctx, ref, test_args, gpu_type, gpus, hours, no_connect, keep):
     TEST_ARGS are passed straight to `python` inside ~/pytorch, e.g.
 
       gpu-dev repro pr/185264 test/inductor/test_flex_attention.py TestFlexAttentionCUDA.test_large_kv_int64_pointer_math_cuda
+
+    --lint runs lintrunner on a CPU box instead (no GPU, no torch build), e.g.
+
+      gpu-dev repro --lint                          # lint main (all files)
+      gpu-dev repro --lint pr/185264                # lint the PR diff (CI-equivalent)
+      gpu-dev repro --lint pr/185264 --all-files    # lint everything
+
+    The box stays up after the run: on a TTY you're dropped in and prompted to
+    cancel on exit (use --keep to leave it running; --no-connect auto-cancels).
     """
     import shlex
     import subprocess
     import sys
+    if not ref:
+        if not lint:
+            rprint("[red]❌ Provide a REF (pr/N, branch, or commit) — or use --lint to lint main.[/red]")
+            sys.exit(2)
+        ref = "main"  # bare `repro --lint` lints current main
+    if not lint and not test_args:
+        rprint("[red]❌ Provide a test, e.g. gpu-dev repro pr/123 test/foo.py — or pass --lint for a lint job.[/red]")
+        sys.exit(2)
+    gpu_type = (gpu_type or ("cpu-x86" if lint else "b200")).lower()
+    if gpu_type.startswith("cpu"):
+        gpus = 0  # CPU reservations must have gpu_count=0
     config = load_config()
     reservation_mgr = ReservationManager(config)
     try:
@@ -1637,9 +1661,37 @@ def repro(ctx, ref, test_args, gpu_type, gpus, hours, no_connect, keep):
         f"PYTHONPATH=/home/dev/pytorch python {testcmd}"
     )
 
+    runlabel, rerun_hint = "test", f"python {testcmd}"
+    if lint:
+        # Lint needs the source tree at the ref but NO torch build. Most pods already
+        # have /home/dev/pytorch; CPU pods may not, so clone (partial) as a fallback.
+        # origin/main is fetched so --merge-base-with works (the PR-diff scope CI lints).
+        # PR ref -> lint the diff (CI-equivalent); main/branch/sha -> lint everything
+        # (merge-base-with origin/main would be empty when you ARE main).
+        lint_default = "--merge-base-with origin/main" if prnum else "--all-files"
+        lintargs = " ".join(shlex.quote(a) for a in test_args) or lint_default
+        runlabel, rerun_hint = "lint", f"lintrunner {lintargs}"
+        remote = (
+            "set -e; "
+            "git config --global --add safe.directory /home/dev/pytorch 2>/dev/null || true; "
+            "if [ ! -d /home/dev/pytorch/.git ]; then echo '[lint] no pytorch tree on this pod — cloning (partial)…'; "
+            "rm -rf /home/dev/pytorch; git clone --filter=blob:none https://github.com/pytorch/pytorch.git /home/dev/pytorch; fi; "
+            "cd /home/dev/pytorch; "
+            + resolve +
+            "echo \"[lint] target ${WANT:-?}\"; "
+            "git fetch origin main 2>/dev/null || true; "
+            "echo \"[lint] checking out $FREF\"; " + checkout + "; "
+            "echo \"[lint] HEAD $(git rev-parse --short HEAD)\"; "
+            "command -v lintrunner >/dev/null 2>&1 || pip install --break-system-packages -q lintrunner; "
+            "echo '[lint] lintrunner init (downloading linters)…'; lintrunner init; "
+            f"echo '[lint] running: lintrunner {lintargs}'; "
+            f"lintrunner {lintargs}"
+        )
+
     # Reserve — warm claim (instant) first, else cold ephemeral. Always no-persist
     # (so the prebuilt tree is staged; a default disk would skip staging).
-    rprint(f"[cyan]🔬 repro: reserving {gpus}x {gpu_type} (warm if available)…[/cyan]")
+    desc = f"{gpus}x {gpu_type}" if gpus else gpu_type
+    rprint(f"[cyan]🔬 repro: reserving {desc} (warm if available)…[/cyan]")
     rid = ssh_cmd = None
     try:
         res = reservation_mgr.claim_direct(
@@ -1675,14 +1727,14 @@ def repro(ctx, ref, test_args, gpu_type, gpus, hours, no_connect, keep):
     except KeyboardInterrupt:
         rprint("\n[yellow]interrupted[/yellow]"); rc = 130
 
-    verdict = "[green]✓ test passed[/green]" if rc == 0 else f"[red]✗ test failed (exit {rc})[/red]"
+    verdict = f"[green]✓ {runlabel} passed[/green]" if rc == 0 else f"[red]✗ {runlabel} failed (exit {rc})[/red]"
 
     # Default (TTY): drop into the box so you can fix and re-run. --no-connect is the
     # CI path: auto-cancel and exit with the test's code.
     connect = (not no_connect) and sys.stdout.isatty()
     if connect:
         rprint(f"\n{verdict} — dropping you into the box at ~/pytorch ({ref} checked out).")
-        rprint(f"[dim]  re-run:  python {testcmd}[/dim]")
+        rprint(f"[dim]  re-run:  {rerun_hint}[/dim]")
         rprint(f"[dim]  finish:  gpu-dev cancel  (from inside)  •  or exit this shell[/dim]\n")
         shell_cmd = f"{ssh_cmd} -t {shlex.quote('cd /home/dev/pytorch 2>/dev/null; exec ${SHELL:-bash} -l')}"
         try:
@@ -1724,6 +1776,47 @@ _SUBMIT_GPU_TYPES = ["b300", "b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g"
                      "a10g", "t4", "l4", "t4-small", "cpu-arm", "cpu-x86", "cpu-spot"]
 
 
+def _build_submit_remote_script(workdir: str, remote_cmd: str, ref: Optional[str],
+                                no_build: bool) -> str:
+    """Build the remote shell script `submit` runs over SSH (under `bash -lc`).
+
+    Without --ref this is just `cd <workdir> && <cmd>`. With --ref the pytorch
+    tree is staged in the *background* in-pod (stage-pytorch &), and the tree is
+    only chowned to dev + the ref fully checked out at the very end. Running the
+    user command before that finishes is the footgun Driss hit: a root-owned tree
+    (git "dubious ownership") and a source/installed-torch mismatch (the ref is
+    checked out but the prebuilt .so is the stale base build -> `import torch`
+    fails). So with --ref we prepend a preamble that:
+      1. waits for staging to finish (`.pytorch-staging` marker removed at end),
+      2. marks /home/dev/pytorch a git safe.directory for the dev user,
+      3. unless --no-build, rebuilds incrementally so installed torch == the
+         checked-out source (warm build/ -> ~tens of seconds; a rebuild failure
+         exits 90 before the user command runs).
+    The rebuild/safe.directory only touch pytorch when staging actually ran
+    (`.pytorch-ready` present), so --disk reservations (ref ignored, no staging)
+    are unaffected.
+    """
+    import shlex
+    cd_run = f"cd {shlex.quote(workdir)} && {remote_cmd}"
+    if not ref:
+        return cd_run
+    lines = [
+        'if [ -e /home/dev/.pytorch-staging ]; then',
+        '  echo "[gpu-dev] waiting for background pytorch --ref staging to finish…"',
+        '  for _i in $(seq 1 3600); do [ -e /home/dev/.pytorch-staging ] || break; sleep 1; done',
+        'fi',
+        'if [ -f /home/dev/.pytorch-ready ]; then',
+        '  git config --global --add safe.directory /home/dev/pytorch 2>/dev/null || true',
+    ]
+    if not no_build:
+        lines += [
+            '  echo "[gpu-dev] rebuilding torch to match --ref (pip install -e . --no-build-isolation)…"',
+            '  ( cd /home/dev/pytorch && pip install -e . --no-build-isolation ) || { echo "[gpu-dev] torch rebuild failed"; exit 90; }',
+        ]
+    lines += ['fi', cd_run]
+    return "\n".join(lines)
+
+
 @main.command(context_settings={"ignore_unknown_options": True})
 @click.option("--gpu-type", type=click.Choice(_SUBMIT_GPU_TYPES, case_sensitive=False), default="a100", show_default=True)
 @click.option("--gpus", type=int, default=1, show_default=True, help="GPU count (multinode if > per-node max).")
@@ -1743,6 +1836,8 @@ _SUBMIT_GPU_TYPES = ["b300", "b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g"
 @click.option("--runtime", type=click.Path(exists=True, file_okay=False, resolve_path=True), default=None,
               help="Local directory to rsync to /workspace/submit-<id>/ on master node before run.")
 @click.option("--no-pull", is_flag=True, help="Skip syncing the remote workspace back to --runtime after the job finishes.")
+@click.option("--no-build", is_flag=True,
+              help="With --ref, skip the incremental torch rebuild before the command (Python-only PRs / quick checks). Default: rebuild so `import torch` reflects the ref.")
 @click.option("--keep-alive", is_flag=True, help="Don't cancel the reservation when the job exits.")
 @click.option("--name", type=str, default=None, help="Reservation name.")
 @click.option("--timeout", type=int, default=24 * 60, show_default=True,
@@ -1750,7 +1845,7 @@ _SUBMIT_GPU_TYPES = ["b300", "b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g"
 @click.argument("command", nargs=-1, required=True)
 @click.pass_context
 def submit(ctx, gpu_type, gpus, hours, disk, ref, no_persistent_disk, spot, dockerfile, dockerimage, preserve_entrypoint,
-           runtime, no_pull, keep_alive, name, timeout, command):
+           runtime, no_pull, no_build, keep_alive, name, timeout, command):
     """Submit a job: reserve, sync code, run, sync results back, auto-cancel.
 
     \b
@@ -1961,11 +2056,15 @@ def submit(ctx, gpu_type, gpus, hours, disk, ref, no_persistent_disk, spot, dock
         else:
             workdir = "/home/dev"
 
-        # Run remote command via login shell so MULTINODE_* etc. are loaded
+        # Run remote command via login shell so MULTINODE_* etc. are loaded. With
+        # --ref, the script first waits for background pytorch staging + rebuilds
+        # so `import torch` matches the checked-out ref (see helper docstring).
         remote_cmd = " ".join(shlex.quote(c) for c in command)
         rprint(f"[cyan]🚀 Running on {ssh_alias}: {remote_cmd}[/cyan]\n")
-        ssh_run = ssh_base + [ssh_alias,
-                              f"cd {shlex.quote(workdir)} && bash -lc {shlex.quote(remote_cmd)}"]
+        if ref and not no_build:
+            rprint("[dim]   (--ref: will wait for staging + rebuild torch first; pass --no-build to skip)[/dim]")
+        remote_script = _build_submit_remote_script(workdir, remote_cmd, ref, no_build)
+        ssh_run = ssh_base + [ssh_alias, f"bash -lc {shlex.quote(remote_script)}"]
         rc = subprocess.call(ssh_run)
         rprint(f"\n[dim]Job exited with code {rc}[/dim]")
 
@@ -3185,6 +3284,172 @@ def show(ctx: click.Context, reservation_id: Optional[str]) -> None:
         rprint(f"[red]❌ Error: {str(e)}[/red]")
 
 
+def _print_recovery_hints(connection_info: dict) -> None:
+    """Tell the user how to unblock/recover their own reservation based on status."""
+    status = (connection_info.get("status") or "").lower()
+    disk_name = connection_info.get("disk_name") or ""
+    res_id = connection_info.get("reservation_id", "") or ""
+    short_id = res_id[:8] if res_id else "<id>"
+    hints = []
+    if status in ("failed", "expired", "cancelled"):
+        if disk_name:
+            hints.append(
+                f"Your data on disk '{disk_name}' is preserved — re-reserve with: "
+                f"gpu-dev reserve --disk {disk_name}")
+            hints.append(f"If that disk is stuck locked: gpu-dev disk unlock {disk_name}")
+        else:
+            hints.append("Re-reserve a new box with: gpu-dev reserve")
+    elif status == "active":
+        hints.append(
+            f"If status is 'active' but you can't SSH, the pod likely died (e.g. OOM). "
+            f"Free it (and your disk) with: gpu-dev cancel {short_id}  — then re-reserve.")
+        if disk_name:
+            hints.append(f"If the disk stays locked after cancel: gpu-dev disk unlock {disk_name}")
+    if hints:
+        rprint("\n[bold]Recovery:[/bold]")
+        for h in hints:
+            rprint(f"  • {h}")
+
+
+def _show_diagnostics(connection_info: dict) -> None:
+    """Render the extra diagnostics `gpu-dev debug` adds on top of the status panel:
+    failure reason, OOM events, the full status-history timeline, captured pod logs,
+    and recovery hints. All sourced from data the lambdas write to DynamoDB, so it
+    needs no cluster/lambda access."""
+    from rich.text import Text
+
+    status = (connection_info.get("status") or "").lower()
+
+    # Failure reason / latest detailed status — shown for ANY status (the normal
+    # `show` only surfaces failure_reason on 'failed'; for an active-but-dead pod
+    # this is exactly what the user needs).
+    failure_reason = (connection_info.get("failure_reason") or "").strip()
+    detailed = (connection_info.get("current_detailed_status") or "").strip()
+    if failure_reason:
+        rprint(f"\n[bold red]Why it ended:[/bold red] {failure_reason}")
+    elif detailed and status != "active":
+        rprint(f"\n[bold]Latest status:[/bold] {detailed}")
+
+    # OOM events
+    oom_count = int(connection_info.get("oom_count", 0) or 0)
+    if oom_count > 0:
+        last = connection_info.get("last_oom_at") or "unknown"
+        cont = connection_info.get("oom_container") or "?"
+        rprint(f"[red]⚠️  OOM:[/red] {oom_count} event(s) — last {last} (container: {cont})")
+
+    # Status-history timeline (the gold for "what happened to my reservation")
+    history = connection_info.get("status_history") or []
+    if history:
+        table = Table(title="Status timeline (most recent last)", show_header=True,
+                      header_style="bold", box=None, pad_edge=False)
+        table.add_column("Time", style="dim", no_wrap=True)
+        table.add_column("Event")
+        for entry in history[-40:]:
+            if isinstance(entry, dict):
+                table.add_row(str(entry.get("timestamp", "")), str(entry.get("message", "")))
+        console.print("")
+        console.print(table)
+    else:
+        rprint("\n[dim]No status history recorded for this reservation.[/dim]")
+
+    # Captured pod logs (lambda snapshot — last lines around the failure)
+    pod_logs = (connection_info.get("pod_logs") or "").strip()
+    if pod_logs:
+        console.print(Panel(Text(pod_logs[-4000:]), title="Captured pod logs (snapshot)",
+                            border_style="yellow"))
+
+    _print_recovery_hints(connection_info)
+
+
+def _show_lambda_logs(reservation_mgr, reservation_id: str, user_id: str) -> None:
+    """Fetch + render the raw lambda (CloudWatch) logs for a reservation."""
+    from rich.text import Text
+    rprint("\n[bold]Fetching lambda logs from CloudWatch…[/bold] [dim](a few seconds)[/dim]")
+    result = reservation_mgr.get_reservation_logs(reservation_id, user_id)
+    if result is None:
+        rprint("[yellow]Could not reach the log backend (it may not be deployed yet, "
+               "or you lack lambda:InvokeFunctionUrl access).[/yellow]")
+        return
+    if result.get("error"):
+        rprint(f"[yellow]Log query: {result['error']}[/yellow]")
+    lines = result.get("lines") or []
+    if not lines:
+        rprint("[dim]No lambda log lines found for this reservation (outside the "
+               "retention window, or none recorded).[/dim]")
+        return
+    body = "\n".join(f"{ln.get('timestamp','')}  {ln.get('message','')}".rstrip()
+                     for ln in lines)
+    console.print(Panel(Text(body[-16000:]),
+                        title=f"Lambda logs · {len(lines)} line(s)", border_style="cyan"))
+
+
+@main.command()
+@click.argument("reservation_id", required=False)
+@click.option("--logs", "show_logs", is_flag=True,
+              help="Also fetch the raw lambda logs for this reservation from CloudWatch.")
+@click.pass_context
+def debug(ctx: click.Context, reservation_id: Optional[str], show_logs: bool) -> None:
+    """Diagnose your own reservation — why a box died or won't connect.
+
+    Shows the status timeline, failure reason, OOM events, and captured pod logs,
+    plus recovery steps — all without needing cluster or lambda access. Add --logs
+    to also pull the raw reservation/expiry lambda logs from CloudWatch.
+
+    \b
+    Examples:
+        gpu-dev debug                 # pick from your active reservations
+        gpu-dev debug abc12345        # a specific reservation (id prefix ok)
+        gpu-dev debug abc12345 --logs # + raw lambda logs from CloudWatch
+
+    For a recently failed/expired box, find its id with 'gpu-dev list' then
+    'gpu-dev debug <id>'.
+    """
+    try:
+        config = load_config()
+        user_info = authenticate_user(config)
+        reservation_mgr = ReservationManager(config)
+
+        # In-pod fast path: the pod's own reservation id is on the env.
+        if reservation_id is None:
+            reservation_id = os.environ.get("GPU_DEV_RESERVATION_ID") or None
+
+        if reservation_id is None:
+            reservations = _fetch_reservations_cross_region(
+                reservation_mgr, user_info["user_id"],
+                ["active", "preparing", "queued", "pending"], config)
+            if not reservations:
+                rprint("[yellow]📋 No active reservations.[/yellow] To debug a recent "
+                       "failed/expired one, find its id with [bold]gpu-dev list[/bold] "
+                       "then run [bold]gpu-dev debug <id>[/bold].")
+                return
+            if len(reservations) == 1:
+                reservation_id = reservations[0].get("reservation_id")
+            else:
+                selected = select_reservation_interactive(reservations, "debug")
+                if not selected or selected in ("__QUIT__", "__ALL__"):
+                    rprint("[yellow]Cancelled.[/yellow]")
+                    return
+                reservation_id = selected
+
+        connection_info = reservation_mgr.get_connection_info(
+            reservation_id, user_info["user_id"])
+        if not connection_info:
+            rprint(f"[red]❌ No reservation found matching '{reservation_id}'[/red] "
+                   "(try a longer id prefix, or check 'gpu-dev list').")
+            return
+
+        _show_single_reservation(connection_info)
+        _show_diagnostics(connection_info)
+        if show_logs:
+            _show_lambda_logs(reservation_mgr, connection_info["reservation_id"],
+                              user_info["user_id"])
+
+    except RuntimeError as e:
+        rprint(f"[red]❌ {str(e)}[/red]")
+    except Exception as e:
+        rprint(f"[red]❌ Error: {str(e)}[/red]")
+
+
 
 def _maybe_show_sdk_tip() -> None:
     """For a user's first few reservations, nudge them toward the Python SDK +

{gpu_dev-0.7.11 → gpu_dev-0.7.13}/cli-tools/gpu-dev-cli/gpu_dev_cli/reservations.py

@@ -613,7 +613,7 @@ class ReservationManager:
                 pass
         return self._direct_url or None
 
-    def _signed_post(self, url: str, payload: dict) -> Optional[dict]:
+    def _signed_post(self, url: str, payload: dict, timeout: int = 20) -> Optional[dict]:
         """SigV4-signed POST to the Function URL. Returns parsed JSON or None."""
         try:
             creds = self.config.session.get_credentials()
@@ -623,13 +623,29 @@ class ReservationManager:
             aws_req = AWSRequest(method="POST", url=url, data=data,
                                  headers={"Content-Type": "application/json"})
             SigV4Auth(creds, "lambda", self.config.aws_region).add_auth(aws_req)
-            resp = requests.post(url, data=data, headers=dict(aws_req.headers), timeout=20)
+            resp = requests.post(url, data=data, headers=dict(aws_req.headers), timeout=timeout)
             if resp.status_code != 200:
                 return None
             return resp.json()
         except Exception:
             return None
 
+    def get_reservation_logs(self, reservation_id: str, user_id: str) -> Optional[Dict[str, Any]]:
+        """Fetch a reservation's lambda logs (CloudWatch Logs Insights) via the
+        processor Function URL. Returns {"lines": [...]} / {"error": ...}, or None if
+        the backend/URL is unavailable. Used by `gpu-dev debug --logs`."""
+        url = self._get_direct_url()
+        if not url:
+            return None
+        payload = {
+            "action": "get_logs",
+            "reservation_id": reservation_id,
+            "user_id": user_id,
+            "version": get_version(),
+        }
+        # CloudWatch Logs Insights queries take longer than a claim — allow ~70s.
+        return self._signed_post(url, payload, timeout=70)
+
     def claim_direct(self, *, user_id: str, gpu_count: int, gpu_type: str,
                      duration_hours: Union[int, float], name: Optional[str] = None,
                      github_user: Optional[str] = None, ref: Optional[str] = None) -> Optional[Dict[str, Any]]:
@@ -999,11 +1015,19 @@ class ReservationManager:
                 "jupyter_enabled": reservation.get("jupyter_enabled", False),
                 "jupyter_error": reservation.get("jupyter_error", ""),
                 "ebs_volume_id": reservation.get("ebs_volume_id", ""),
+                "disk_name": reservation.get("disk_name", ""),
                 "secondary_users": reservation.get("secondary_users", []),
                 "warning": reservation.get("warning", ""),
                 "is_multinode": is_multinode,
                 "pod_ip": reservation.get("pod_ip", ""),
+                "node_ip": reservation.get("node_ip", ""),
+                "node_name": reservation.get("node_name", ""),
                 "fqdn": reservation.get("fqdn", ""),
+                # Health/diagnostics (surfaced by `gpu-dev debug`); written by the
+                # reservation + expiry lambdas. Present off the raw item, not always set.
+                "oom_count": int(reservation.get("oom_count", 0) or 0),
+                "last_oom_at": reservation.get("last_oom_at", ""),
+                "oom_container": reservation.get("oom_container", ""),
             }
 
             # If multi-node, fetch all nodes in the group

gpu_dev-0.7.13/docs/GPU_DEV_SUBMIT.md

@@ -0,0 +1,89 @@
+# `gpu-dev submit` — guide & footguns
+
+`gpu-dev submit` reserves a box, (optionally) rsyncs a local dir up, runs your
+command over SSH, syncs results back, and auto-cancels. It's the non-interactive
+sibling of `gpu-dev reserve` — good for CI-style validation, one-shot test runs,
+and scripted repros.
+
+```bash
+# run a script in a local dir on 1x H100, sync results back, auto-cancel
+gpu-dev submit --runtime ./ --gpu-type h100 -- bash run.sh
+
+# validate a PyTorch PR's tests on H100 (stages + builds the PR for you)
+gpu-dev submit --gpu-type h100 --no-persistent-disk --ref pr/186015 -- \
+    python test/test_foo.py -k some_test
+
+# keep the box after the job (debug a failure interactively)
+gpu-dev submit --keep-alive --gpu-type h100 -- pytest test/test_x.py
+```
+
+Exit code = your command's exit code (so it composes in scripts/CI).
+
+---
+
+## Footguns (read before your first `--ref` run)
+
+### 1. `--ref` stages PyTorch in the background — `submit` now waits for it
+With `--ref`, the in-pod startup checks out your ref into `/home/dev/pytorch`
+**in the background** and only chowns the tree to `dev` + finishes the checkout
+at the very end. Historically `submit` could SSH in and run your command before
+that finished, so you'd hit:
+- a **root-owned** `/home/dev/pytorch` (git: *"detected dubious ownership"*), and
+- a **source/installed-torch mismatch** → `import torch` fails (the ref source is
+  checked out but the importable `.so` is still the stale prebuilt base).
+
+`submit` now **waits for staging to complete**, marks the tree a git
+`safe.directory`, and (by default) **rebuilds incrementally** so the installed
+torch matches the checked-out ref before your command runs. You don't need the
+`sudo chown` / `safe.directory` workaround anymore.
+
+### 2. `--ref` rebuilds torch by default — use `--no-build` to skip
+The dropped-in `build/` + `.so` come from the **base** tree, not your ref. To make
+`import torch` reflect your ref's compiled (C++/CUDA) changes, `submit --ref`
+runs `pip install -e . --no-build-isolation` (incremental, warm `build/` →
+typically tens of seconds; a cold/cross-arch build is much longer).
+
+- Pass **`--no-build`** for Python-only PRs or quick checks — skips the rebuild
+  (import still works; it just won't include compiled changes).
+- A rebuild failure exits **90** *before* your command runs (so a broken build
+  doesn't masquerade as a test failure).
+
+### 3. Prebuilt fast path is **prod-arch only** (H100 / B200)
+The by-SHA / viable-strict prebuilt trees are compiled for `sm_90;sm_100`
+(H100/B200). On other GPU types (t4, a100, l4, …) or staging there's no matching
+prebuilt, so `--ref` falls back to a **full from-scratch build** — slow. Validate
+ref-based jobs on `--gpu-type h100` (or `b200`).
+
+### 4. `--ref` is ignored with `--disk`
+A persistent disk brings its own `/home/dev/pytorch`; `--ref` does **not** stage
+onto a `--disk` reservation (and `submit` won't rebuild it). Use
+`--no-persistent-disk` (or omit `--disk`) when you want a ref staged.
+
+### 5. `--preserve-entrypoint` needs SSH
+`submit` runs your command over SSH, so a custom image with
+`--preserve-entrypoint` must still expose the SSH harness or `submit` can't reach
+it. For pure entrypoint containers, use `reserve`, not `submit`.
+
+### 6. Results sync-back is best-effort
+With `--runtime`, output is rsync'd back to your local dir when the job exits
+(unless `--no-pull`). If the box dies mid-job (spot reclaim, expiry) the sync-back
+may be partial — you'll see a warning. For long jobs prefer `--keep-alive` and
+pull manually, or write important artifacts to `/shared-personal` (persists
+across reservations).
+
+### 7. `--hours` is a ceiling, not the runtime
+It's the reservation lifetime cap; the job auto-cancels as soon as your command
+exits (unless `--keep-alive`). Set it high enough that queueing + build + run fit.
+
+---
+
+## Finding footguns early
+
+- `gpu-dev submit --keep-alive … -- true` then `gpu-dev connect <id>` — get a
+  box in the exact submit state and poke around before committing a real run.
+- With `--ref`, watch staging directly: `tail -f /home/dev/.pytorch-staging.log`
+  in the pod; `.pytorch-ready` (HEAD sha) is written when staging is done.
+- `python -c "import torch; print(torch.__file__, torch.version.git_version)"`
+  confirms which torch you're actually importing vs. the ref you asked for.
+
+Found a new one? Add it here and ping `oncall:pytorch_release_engineering`.

{gpu_dev-0.7.11 → gpu_dev-0.7.13}/gpu_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.7.11
+Version: 0.7.13
 Summary: CLI + Python SDK for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.7.11 → gpu_dev-0.7.13}/gpu_dev.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ cli-tools/gpu-dev-cli/gpu_dev_cli/reservations.py
 cli-tools/gpu-dev-cli/gpu_dev_cli/ssh_proxy.py
 cli-tools/scripts/clear_stale_disk_locks.py
 docs/FAST_REPRO_DESIGN.md
+docs/GPU_DEV_SUBMIT.md
 docs/SDK_REPRO.md
 docs/USER_GUIDE.md
 docs/devgpu-features.html
@@ -197,6 +198,7 @@ tests/unit/cli/test_cancel.py
 tests/unit/cli/test_config_cmd.py
 tests/unit/cli/test_config_module.py
 tests/unit/cli/test_connect.py
+tests/unit/cli/test_debug.py
 tests/unit/cli/test_disks.py
 tests/unit/cli/test_edit.py
 tests/unit/cli/test_interactive.py
@@ -212,6 +214,9 @@ tests/unit/lambda_fn/__init__.py
 tests/unit/lambda_fn/test_availability.py
 tests/unit/lambda_fn/test_cancellation.py
 tests/unit/lambda_fn/test_claim.py
+tests/unit/lambda_fn/test_dead_pod_cleanup.py
+tests/unit/lambda_fn/test_finalize_no_ssh.py
+tests/unit/lambda_fn/test_get_logs.py
 tests/unit/lambda_fn/test_mig_gpu_config.py
 tests/unit/lambda_fn/test_pod_resources.py
 tests/unit/lambda_fn/test_ref_staging.py

{gpu_dev-0.7.11 → gpu_dev-0.7.13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gpu-dev"
-version = "0.7.11"
+version = "0.7.13"
 description = "CLI + Python SDK for PyTorch GPU developer server reservations"
 authors = [{name = "PyTorch Team"}]
 readme = "cli-tools/gpu-dev-cli/README.md"