gpu-dev 0.5.24__tar.gz → 0.5.26__tar.gz

{gpu_dev-0.5.24 → gpu_dev-0.5.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.5.24
+Version: 0.5.26
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.5.24 → gpu_dev-0.5.26}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.5.24
+Version: 0.5.26
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.5.24 → gpu_dev-0.5.26}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/SOURCES.txt

@@ -52,6 +52,7 @@ terraform-gpu-devservers/main.tf
 terraform-gpu-devservers/mig-config.tf
 terraform-gpu-devservers/mig-parted-config.yaml
 terraform-gpu-devservers/monitoring.tf
+terraform-gpu-devservers/node-termination-handler.tf
 terraform-gpu-devservers/outputs.tf
 terraform-gpu-devservers/pyproject.toml
 terraform-gpu-devservers/queue.tf

{gpu_dev-0.5.24 → gpu_dev-0.5.26}/cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py

@@ -4,6 +4,7 @@ Reserve and manage GPU development servers
 """
 
 import click
+import os
 from typing import Optional
 from rich.console import Console
 from rich.table import Table
@@ -3218,6 +3219,34 @@ def connect(ctx: click.Context, reservation_id: Optional[str]) -> None:
         if "-A" not in ssh_command and "-o ForwardAgent=yes" not in ssh_command:
             ssh_command = ssh_command.replace("ssh ", "ssh -A ", 1)
 
+        # Inject AddKeysToAgent so the first connect from this laptop loads the user\'s
+        # IdentityFile into ssh-agent — without this the forwarded agent is empty on
+        # subsequent pod→pod hops. UseKeychain persists the passphrase across reboots on
+        # macOS; IgnoreUnknown lets Linux SSH ignore the macOS-only option cleanly.
+        # The same options live in ~/.gpu-dev/<id>-sshconfig but ssh only honours them
+        # when the command-line target matches a Host block, which this connect command
+        # bypasses by passing the FQDN directly.
+        if "AddKeysToAgent" not in ssh_command:
+            ssh_command = ssh_command.replace(
+                "ssh ",
+                "ssh -o AddKeysToAgent=yes -o IgnoreUnknown=UseKeychain -o UseKeychain=yes ",
+                1,
+            )
+
+        # When running from inside a gpu-dev pod (=GPU_DEV_USER_ID env var set) and the
+        # forwarded SSH agent is reachable but empty, the next hop is going to fail with
+        # 'Permission denied (publickey)'. Warn upfront so the user knows to ssh-add on
+        # their laptop instead of debugging an opaque auth failure on the remote side.
+        if os.environ.get("GPU_DEV_USER_ID") and os.environ.get("SSH_AUTH_SOCK"):
+            try:
+                import subprocess as _sp
+                r = _sp.run(["ssh-add", "-L"], capture_output=True, text=True, timeout=3)
+                if r.returncode != 0 or not r.stdout.strip() or "no identities" in r.stdout.lower():
+                    rprint("[yellow]⚠️  Forwarded SSH agent is empty — second-hop SSH from a pod will fail auth.[/yellow]")
+                    rprint("[yellow]   On your laptop: `ssh-add ~/.ssh/id_ed25519` (or your GitHub key), then reconnect to this pod with `gpu-dev connect`.[/yellow]\n")
+            except Exception:
+                pass
+
         # Parse and execute the command, capturing exit code for auth failures
         rprint(f"[dim]Executing: {ssh_command}[/dim]\n")
         result = subprocess.run(ssh_command, shell=True)

{gpu_dev-0.5.24 → gpu_dev-0.5.26}/cli-tools/gpu-dev-cli/gpu_dev_cli/config.py

@@ -22,6 +22,11 @@ class Config:
             "workspace": "prod",
             "description": "Production environment",
         },
+        "prod-east1": {
+            "region": "us-east-1",
+            "workspace": "prod-east1",
+            "description": "Spot-only us-east-1 environment (T4/L4/CPU)",
+        },
     }
     DEFAULT_ENVIRONMENT = "prod"
 

{gpu_dev-0.5.24 → gpu_dev-0.5.26}/cli-tools/gpu-dev-cli/gpu_dev_cli/reservations.py

@@ -162,11 +162,20 @@ def _generate_ssh_config(hostname: str, pod_name: str) -> str:
     Returns:
         SSH config content as string
     """
+    import sys
+    # AddKeysToAgent makes SSH stash the IdentityFile into ssh-agent the first time it
+    # uses it for this Host, so the next agent-forwarding hop (pod → pod) actually has
+    # something to forward. On macOS, UseKeychain persists the passphrase via Keychain
+    # so users aren\'t prompted on every shell restart. Linux ssh errors on UseKeychain,
+    # so guard it with IgnoreUnknown.
+    extra = "    AddKeysToAgent yes\n"
+    if sys.platform == "darwin":
+        extra += "    IgnoreUnknown UseKeychain\n    UseKeychain yes\n"
     config_content = f"""Host {pod_name}
     HostName {hostname}
     User dev
     ForwardAgent yes
-    ProxyCommand gpu-dev-ssh-proxy %h %p
+{extra}    ProxyCommand gpu-dev-ssh-proxy %h %p
     StrictHostKeyChecking no
     UserKnownHostsFile /dev/null
 """

{gpu_dev-0.5.24 → gpu_dev-0.5.26}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gpu-dev"
-version = "0.5.24"
+version = "0.5.26"
 description = "CLI tool for PyTorch GPU developer server reservations"
 authors = [{name = "PyTorch Team"}]
 readme = "cli-tools/gpu-dev-cli/README.md"

{gpu_dev-0.5.24 → gpu_dev-0.5.26}/terraform-gpu-devservers/eks.tf

@@ -401,11 +401,14 @@ resource "aws_launch_template" "gpu_dev_launch_template" {
     }
   }
 
-  # Conditionally add instance_market_options for capacity block instances (only when capacity reservation exists)
+  # instance_market_options: capacity-block when bound to a reservation, spot when
+  # the workspace's gpu_config has use_spot=true, otherwise on-demand (no block).
+  # Spot is mutually exclusive with capacity reservations — AWS rejects launch templates
+  # carrying both, so the precedence here is CR > spot > on-demand.
   dynamic "instance_market_options" {
-    for_each = each.value.capacity_reservation_id != null ? [1] : []
+    for_each = (each.value.capacity_reservation_id != null || try(each.value.gpu_config.use_spot, false)) ? [1] : []
     content {
-      market_type = "capacity-block"
+      market_type = each.value.capacity_reservation_id != null ? "capacity-block" : "spot"
     }
   }
 

{gpu_dev-0.5.24 → gpu_dev-0.5.26}/terraform-gpu-devservers/main.tf

@@ -58,6 +58,13 @@ provider "helm" {
 # Data sources
 data "aws_availability_zones" "available" {
   state = "available"
+  # Exclude Local Zones (e.g. us-east-1-dfw-2a) and Wavelength Zones — EKS control
+  # plane only supports standard AZs. us-east-2 doesn't have Local Zones so the
+  # existing prod workspace was unaffected; us-east-1 has several (dfw, bos, …).
+  filter {
+    name   = "opt-in-status"
+    values = ["opt-in-not-required"]
+  }
 }
 
 data "aws_caller_identity" "current" {}
@@ -315,6 +322,50 @@ locals {
         }
       }
     }
+    # us-east-1 spot-only experimental cluster.
+    # Same provisioning shape as prod (managed via the terraform.workspace switch) but
+    # backed entirely by EC2 Spot — first cheap-and-cheerful environment we can deploy
+    # new instance types into (B300 land here once on-demand quota arrives).
+    "prod-east1" = {
+      aws_region             = "us-east-1"
+      environment            = "prod-east1"
+      domain_name            = "east1.devservers.io"
+      gpu_instance_count     = 1
+      use_self_managed_nodes = true
+      instance_type          = "g4dn.12xlarge"
+      supported_gpu_types = {
+        "t4" = {
+          instance_type       = "g4dn.12xlarge"
+          instance_types      = null
+          instance_count      = 1
+          gpus_per_instance   = 4
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 0
+          use_spot            = true
+        }
+        "l4" = {
+          instance_type       = "g6.12xlarge"
+          instance_types      = null
+          instance_count      = 1
+          gpus_per_instance   = 4
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 1
+          use_spot            = true
+        }
+        "cpu-x86" = {
+          instance_type       = "c7i.8xlarge"
+          instance_types      = null
+          instance_count      = 5
+          gpus_per_instance   = 0
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 0
+          use_spot            = true
+        }
+      }
+    }
   }
 
   # Current workspace configuration
@@ -322,6 +373,9 @@ locals {
 
   # Workspace-specific capacity reservations (with manual instance counts)
   capacity_reservations = {
+    "prod-east1" = {
+      # No capacity reservations — this workspace is spot-only.
+    }
     default = {
       # Test environment capacity reservations
       # h100 = [
@@ -366,6 +420,13 @@ locals {
 
   # Workspace-specific GPU type to subnet mappings
   gpu_subnet_assignments = {
+    "prod-east1" = {
+      # All node types land in the primary subnet (us-east-1a). Spot availability is
+      # better than placement-group-strictness on these small ASGs.
+      t4         = "primary"
+      l4         = "primary"
+      "cpu-x86"  = "primary"
+    }
     default = {
       # Test environment - T4 nodes in multiple AZs for testing
       t4         = "primary"   # T4 in us-west-1a (primary AZ)
@@ -392,6 +453,9 @@ locals {
 
   # Per-capacity-reservation AZ mappings (overrides gpu_subnet_assignments when CR is used)
   capacity_reservation_azs = {
+    "prod-east1" = {
+      # Empty — no CRs in this workspace.
+    }
     default = {
       "cr-04d3d1d84e127a562" = "secondary" # us-west-1c
     }

gpu_dev-0.5.26/terraform-gpu-devservers/node-termination-handler.tf

@@ -0,0 +1,36 @@
+# AWS Node Termination Handler — graceful drain on spot-interrupt + ASG lifecycle events.
+#
+# IMDS mode (one DaemonSet per node, no SQS / no IAM role) is plenty for our use case:
+# we don't care about queue-processor features (rebalance recommendations, scheduled
+# events). We just want pods to get a clean SIGTERM when AWS sends the 2-minute spot
+# notice via instance metadata, instead of being killed cold.
+#
+# Tolerates everything so it runs on the GPU nodes that have nvidia.com/gpu:NoSchedule.
+
+resource "helm_release" "aws_node_termination_handler" {
+  name             = "aws-node-termination-handler"
+  repository       = "https://aws.github.io/eks-charts"
+  chart            = "aws-node-termination-handler"
+  namespace        = "kube-system"
+  version          = "0.27.1"
+  cleanup_on_fail  = true
+
+  values = [yamlencode({
+    enableSpotInterruptionDraining = true
+    enableScheduledEventDraining   = true
+    enableRebalanceMonitoring      = true
+    enableRebalanceDraining        = false # warning only; rebalance recommendations are too noisy
+    nodeSelector = {
+      "kubernetes.io/os" = "linux"
+    }
+    tolerations = [
+      { operator = "Exists" }, # tolerate every taint; we want NTH on every node, including GPU nodes
+    ]
+    resources = {
+      requests = { cpu = "50m", memory = "64Mi" }
+      limits   = { cpu = "100m", memory = "128Mi" }
+    }
+  })]
+
+  depends_on = [aws_eks_cluster.gpu_dev_cluster]
+}