gpu-dev 0.5.26__tar.gz → 0.5.28__tar.gz

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.5.26
+Version: 0.5.28
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.5.26
+Version: 0.5.28
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py

@@ -496,7 +496,7 @@ def main(ctx: click.Context) -> None:
     "--gpu-type",
     "-t",
     type=click.Choice(
-        ["b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g", "h200", "h100", "h100-mig-1g", "h100-mig-2g", "h100-mig-3g", "a100", "rtxpro6000", "a10g", "t4", "l4", "t4-small", "cpu-arm", "cpu-x86"], case_sensitive=False
+        ["b300", "b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g", "h200", "h100", "h100-mig-1g", "h100-mig-2g", "h100-mig-3g", "a100", "rtxpro6000", "a10g", "t4", "l4", "t4-small", "cpu-arm", "cpu-x86"], case_sensitive=False
     ),
     help="GPU type to reserve. Full GPUs: b200, h200, h100, a100, rtxpro6000, a10g, t4, l4, t4-small. H100 MIG slices: h100-mig-1g (10 GB), h100-mig-2g (20 GB), h100-mig-3g (40 GB). B200 MIG slices (on the mixed B200 node): b200-mig-1g (23 GB), b200-mig-2g (45 GB), b200-mig-3g (90 GB). CPU: cpu-arm, cpu-x86.",
 )
@@ -576,6 +576,8 @@ def main(ctx: click.Context) -> None:
     multiple=True,
     help="Request nodes with specific label (format: key=value). Example: --node-label nsight=true for Nsight profiling nodes",
 )
+@click.option("--spot", is_flag=True, default=False,
+              help="Acknowledge spot instance (~1/3 cost, may be preempted with 2-min notice). Required for spot-only types.")
 @click.pass_context
 def reserve(
     ctx: click.Context,
@@ -662,6 +664,7 @@ def reserve(
             "b200-mig-3g": {"max_gpus": 2, "instance_type": "p6-b200.48xlarge"},
             "h200": {"max_gpus": 8, "instance_type": "p5e.48xlarge"},
             "b200": {"max_gpus": 8, "instance_type": "p6-b200.48xlarge"},
+            "b300": {"max_gpus": 8, "instance_type": "p6-b300.48xlarge"},
             "cpu-arm": {"max_gpus": 0, "instance_type": "c7g.4xlarge"},
             "cpu-x86": {"max_gpus": 0, "instance_type": "c7i.4xlarge"},
         }
@@ -1270,6 +1273,7 @@ def reserve(
                     no_persistent_disk=no_persistent_disk,
                     preserve_entrypoint=preserve_entrypoint,
                     disk_name=disk,
+                    spot=spot,
                     node_labels=node_labels if node_labels else None,
                 )
             else:
@@ -1288,6 +1292,7 @@ def reserve(
                     no_persistent_disk=no_persistent_disk,
                     preserve_entrypoint=preserve_entrypoint,
                     disk_name=disk,
+                    spot=spot,
                     node_labels=node_labels if node_labels else None,
                     trace=trace,
                 )
@@ -1350,7 +1355,7 @@ def reserve(
         rprint(f"[red]❌ Error: {str(e)}[/red]")
 
 
-_SUBMIT_GPU_TYPES = ["b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g", "h200", "h100",
+_SUBMIT_GPU_TYPES = ["b300", "b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g", "h200", "h100",
                      "h100-mig-1g", "h100-mig-2g", "h100-mig-3g", "a100", "rtxpro6000",
                      "a10g", "t4", "l4", "t4-small", "cpu-arm", "cpu-x86"]
 
@@ -1361,6 +1366,8 @@ _SUBMIT_GPU_TYPES = ["b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g", "h200"
 @click.option("--hours", type=float, default=1.0, show_default=True, help="Reservation lifetime ceiling — job auto-cancels well before this if it finishes.")
 @click.option("--disk", type=str, default=None, help="Persistent disk name (master node only). Omit for ephemeral storage.")
 @click.option("--no-persistent-disk", is_flag=True, help="Skip persistent disk entirely.")
+@click.option("--spot", is_flag=True, default=False,
+              help="Acknowledge spot instance (~1/3 cost, may be preempted). Required for spot-only types.")
 @click.option("--dockerfile", type=click.Path(exists=True, dir_okay=False, resolve_path=True), default=None,
               help="Local Dockerfile to build into the pod image (build context = the Dockerfile's directory).")
 @click.option("--dockerimage", type=str, default=None,
@@ -1376,7 +1383,7 @@ _SUBMIT_GPU_TYPES = ["b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g", "h200"
               help="Minutes to wait for the reservation to become active. Defaults to 24h since GPU reservations may queue when the cluster is full.")
 @click.argument("command", nargs=-1, required=True)
 @click.pass_context
-def submit(ctx, gpu_type, gpus, hours, disk, no_persistent_disk, dockerfile, dockerimage, preserve_entrypoint,
+def submit(ctx, gpu_type, gpus, hours, disk, no_persistent_disk, spot, dockerfile, dockerimage, preserve_entrypoint,
            runtime, no_pull, keep_alive, name, timeout, command):
     """Submit a job: reserve, sync code, run, sync results back, auto-cancel.
 
@@ -1490,7 +1497,7 @@ def submit(ctx, gpu_type, gpus, hours, disk, no_persistent_disk, dockerfile, doc
             user_id=user_info["user_id"], gpu_count=gpus, gpu_type=gt,
             duration_hours=hours, name=name, github_user=user_info["github_user"],
             no_persistent_disk=no_persistent_disk, disk_name=disk_name,
-            dockerfile=dockerfile_payload, dockerimage=dockerimage,
+            spot=spot, dockerfile=dockerfile_payload, dockerimage=dockerimage,
             preserve_entrypoint=preserve_entrypoint)
         if not reservation_ids:
             rprint("[red]❌ Failed to create multinode reservation[/red]")
@@ -1501,7 +1508,7 @@ def submit(ctx, gpu_type, gpus, hours, disk, no_persistent_disk, dockerfile, doc
             user_id=user_info["user_id"], gpu_count=gpus, gpu_type=gt,
             duration_hours=hours, name=name, github_user=user_info["github_user"],
             no_persistent_disk=no_persistent_disk, disk_name=disk_name,
-            dockerfile=dockerfile_payload, dockerimage=dockerimage,
+            spot=spot, dockerfile=dockerfile_payload, dockerimage=dockerimage,
             preserve_entrypoint=preserve_entrypoint)
         if not primary_id:
             rprint("[red]❌ Failed to create reservation[/red]")
@@ -2719,6 +2726,7 @@ def _show_availability() -> None:
             # GPU architecture mapping (for display)
             gpu_architectures = {
                 "b200": "Blackwell (sm100)",
+                "b300": "Blackwell (sm100)",
                 "h200": "Hopper (sm90)",
                 "h100": "Hopper (sm90)",
                 "a100": "Ampere (sm80)",
@@ -2880,6 +2888,7 @@ def _show_availability_watch(interval: int) -> None:
                         # GPU architecture mapping (for display)
                         gpu_architectures = {
                             "b200": "Blackwell (sm100)",
+                "b300": "Blackwell (sm100)",
                             "h200": "Hopper (sm90)",
                             "h100": "Hopper (sm90)",
                             "a100": "Ampere (sm80)",

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/cli-tools/gpu-dev-cli/gpu_dev_cli/reservations.py

@@ -421,6 +421,7 @@ class ReservationManager:
         disk_name: Optional[str] = None,
         node_labels: Optional[Dict[str, str]] = None,
         trace: bool = False,
+        spot: bool = False,
     ) -> Optional[str]:
         """Create a new GPU reservation"""
         try:
@@ -500,6 +501,9 @@ class ReservationManager:
             if node_labels:
                 message["node_labels"] = node_labels
 
+            if spot:
+                message["spot"] = True
+
             # Add trace flag and CLI start timestamp
             if trace:
                 message["trace"] = True
@@ -536,6 +540,7 @@ class ReservationManager:
         preserve_entrypoint: bool = False,
         disk_name: Optional[str] = None,
         node_labels: Optional[Dict[str, str]] = None,
+        spot: bool = False,
     ) -> Optional[List[str]]:
         """Create multiple GPU reservations for multinode setup"""
         try:
@@ -557,6 +562,7 @@ class ReservationManager:
                 "b200-mig-3g": {"max_gpus": 2},
                 "h200": {"max_gpus": 8},
                 "b200": {"max_gpus": 8},
+                "b300": {"max_gpus": 8},
             }
 
             max_gpus_per_node = gpu_configs[gpu_type]["max_gpus"]
@@ -601,6 +607,7 @@ class ReservationManager:
                     "recreate_env": recreate_env,
                     "is_multinode": True,
                     "no_persistent_disk": no_persistent_disk,
+                    "spot": spot,
                 }
 
                 if github_user:

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gpu-dev"
-version = "0.5.26"
+version = "0.5.28"
 description = "CLI tool for PyTorch GPU developer server reservations"
 authors = [{name = "PyTorch Team"}]
 readme = "cli-tools/gpu-dev-cli/README.md"

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/terraform-gpu-devservers/eks.tf

@@ -189,6 +189,7 @@ locals {
     "h100"     = "h100"
     "h200"     = "h200"
     "b200"     = "b200"
+    "b300"     = "b300"
     "a100"     = "a100"
     "cpu-arm"  = "cpu-arm"
     "cpu-x86"  = "cpu-x86"

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/terraform-gpu-devservers/lambda/reservation_processor/index.py

@@ -59,6 +59,7 @@ ECR_REPOSITORY_URL = os.environ.get("ECR_REPOSITORY_URL")
 # Version validation - injected via Terraform
 LAMBDA_VERSION = os.environ.get("LAMBDA_VERSION", "0.3.9")
 MIN_CLI_VERSION = os.environ.get("MIN_CLI_VERSION", "0.3.9")
+SPOT_GPU_TYPES = os.environ.get("SPOT_GPU_TYPES", "")
 OPERATIONS_TABLE = os.environ.get("OPERATIONS_TABLE", "pytorch-gpu-dev-operations")
 
 # GPU Configuration - single source of truth for all GPU type mappings
@@ -81,6 +82,7 @@ GPU_CONFIG = {
     "h100": {"instance_type": "p5.48xlarge", "max_gpus": 8, "cpus": 192, "memory_gb": 2048, "efa_count": 32},
     "h200": {"instance_type": "p5e.48xlarge", "max_gpus": 8, "cpus": 192, "memory_gb": 2048, "efa_count": 32},
     "b200": {"instance_type": "p6-b200.48xlarge", "max_gpus": 8, "cpus": 192, "memory_gb": 2048, "efa_count": 32},
+    "b300": {"instance_type": "p6-b300.48xlarge", "max_gpus": 8, "cpus": 192, "memory_gb": 2048, "efa_count": 8},
     "cpu-arm": {"instance_type": "c7g.8xlarge", "max_gpus": 0, "cpus": 32, "memory_gb": 64, "efa_count": 0},
     "cpu-x86": {"instance_type": "c7i.8xlarge", "max_gpus": 0, "cpus": 32, "memory_gb": 64, "efa_count": 0},
 }
@@ -2188,7 +2190,7 @@ def validate_reservation_request(request: dict[str, Any]) -> tuple[bool, str]:
     # Validate GPU type
     valid_gpu_types = ["t4", "l4", "a10g", "rtxpro6000", "t4-small", "a100",
                        "h100", "h100-mig-1g", "h100-mig-2g", "h100-mig-3g",
-                       "h200", "b200", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g",
+                       "h200", "b200", "b300", "b200-mig-1g", "b200-mig-2g", "b200-mig-3g",
                        "cpu-arm", "cpu-x86"]
     if gpu_type not in valid_gpu_types:
         error_msg = f"Invalid GPU type: {gpu_type}. Must be one of: {', '.join(valid_gpu_types)}"
@@ -2205,6 +2207,19 @@ def validate_reservation_request(request: dict[str, Any]) -> tuple[bool, str]:
             logger.warning(f"User {user_id} blocked from {gpu_type}: maintenance mode")
             return False, error_msg
 
+    # Spot acknowledgment: if this workspace marks the GPU type as spot-only and
+    # the user didn't pass --spot, reject with a clear message.
+    if SPOT_GPU_TYPES and not request.get("spot", False):
+        is_spot = SPOT_GPU_TYPES.strip() == "all" or gpu_type in [t.strip() for t in SPOT_GPU_TYPES.split(",")]
+        if is_spot:
+            error_msg = (
+                f"{gpu_type.upper()} is only available as a spot instance in this environment. "
+                f"Spot instances are ~1/3 the cost but can be reclaimed by AWS with 2-min notice. "
+                f"Pass --spot to confirm: gpu-dev reserve --gpu-type {gpu_type} --spot"
+            )
+            logger.warning(f"Reservation: spot acknowledgment missing for {gpu_type}")
+            return False, error_msg
+
     # Validate GPU count based on type
     if gpu_type.startswith("cpu-") and gpu_count == 0:
         pass  # Valid CPU-only instance
@@ -2435,6 +2450,7 @@ def update_gpu_availability_table(
             "b200-mig-3g": {"gpus_per_instance": 2},
             "h200": {"gpus_per_instance": 8},
             "b200": {"gpus_per_instance": 8},
+            "b300": {"gpus_per_instance": 8},
         }
 
         gpu_config = gpu_type_configs.get(gpu_type, {"gpus_per_instance": 8})
@@ -6529,6 +6545,7 @@ def get_instance_type_and_gpu_info(k8s_client, pod_name: str) -> tuple[str, str]
             "p5e.48xlarge": "H200",
             "p5en.48xlarge": "H200",
             "p6-b200.48xlarge": "B200",
+            "p6-b300.48xlarge": "B300",
         }
 
         gpu_type = gpu_type_mapping.get(instance_type, "Unknown")

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/terraform-gpu-devservers/lambda.tf

@@ -180,8 +180,13 @@ resource "aws_lambda_function" "reservation_processor" {
       HOSTED_ZONE_ID                     = local.effective_domain_name != "" ? local.hosted_zone_id : ""
       SSH_DOMAIN_MAPPINGS_TABLE          = local.effective_domain_name != "" ? aws_dynamodb_table.ssh_domain_mappings.name : ""
       SSL_CERTIFICATE_ARN                = local.effective_domain_name != "" ? aws_acm_certificate.wildcard[0].arn : ""
-      LAMBDA_VERSION                     = "0.5.25"
+      LAMBDA_VERSION                     = "0.5.28"
       MIN_CLI_VERSION                    = "0.5.16"
+      # Comma-separated GPU types that require --spot flag, or "all" for every type.
+      # Empty = no spot types (on-demand / reserved). Set per-workspace.
+      SPOT_GPU_TYPES                     = lookup({
+        "prod-east1" = "all"
+      }, terraform.workspace, "")
       DISK_CONTENTS_BUCKET               = aws_s3_bucket.disk_contents.bucket
       OPERATIONS_TABLE                   = aws_dynamodb_table.operations.name
     }, local.alb_env_vars)

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/terraform-gpu-devservers/main.tf

@@ -334,6 +334,60 @@ locals {
       use_self_managed_nodes = true
       instance_type          = "g4dn.12xlarge"
       supported_gpu_types = {
+        # 8-GPU spot instances. instance_count=1 means the ASG tries to maintain 1
+        # spot instance per type — if AWS can't grant it (capacity / quota), the ASG
+        # sits at 0 and gpu-dev reservations queue. Bump counts once we see what
+        # actually gets fulfilled in us-east-1.
+        "b300" = {
+          instance_type       = "p6-b300.48xlarge"
+          instance_types      = null
+          instance_count      = 1
+          gpus_per_instance   = 8
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 8
+          use_spot            = true
+        }
+        "b200" = {
+          instance_type       = "p6-b200.48xlarge"
+          instance_types      = null
+          instance_count      = 1
+          gpus_per_instance   = 8
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 8
+          use_spot            = true
+        }
+        "h200" = {
+          instance_type       = "p5e.48xlarge"
+          instance_types      = null
+          instance_count      = 1
+          gpus_per_instance   = 8
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 16
+          use_spot            = true
+        }
+        "h100" = {
+          instance_type       = "p5.48xlarge"
+          instance_types      = null
+          instance_count      = 1
+          gpus_per_instance   = 8
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 32
+          use_spot            = true
+        }
+        "a100" = {
+          instance_type       = "p4d.24xlarge"
+          instance_types      = null
+          instance_count      = 1
+          gpus_per_instance   = 8
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 4
+          use_spot            = true
+        }
         "t4" = {
           instance_type       = "g4dn.12xlarge"
           instance_types      = null
@@ -421,8 +475,15 @@ locals {
   # Workspace-specific GPU type to subnet mappings
   gpu_subnet_assignments = {
     "prod-east1" = {
-      # All node types land in the primary subnet (us-east-1a). Spot availability is
-      # better than placement-group-strictness on these small ASGs.
+      # All node types land in the primary subnet (us-east-1a). Multi-EFA types
+      # (efa_network_cards > 1) automatically use the private subnet in the same AZ.
+      # Specific instance types may not have capacity in us-east-1a — those ASGs will
+      # sit at 0 until we widen to other AZs, that's expected for beta.
+      b300       = "primary"
+      b200       = "primary"
+      h200       = "primary"
+      h100       = "primary"
+      a100       = "primary"
       t4         = "primary"
       l4         = "primary"
       "cpu-x86"  = "primary"
@@ -451,6 +512,22 @@ locals {
     }
   }
 
+  # Subdomain NS delegations to create in *this* workspace's parent zone. Lets
+  # prod (which owns devservers.io) auto-publish NS records pointing at child zones
+  # in other workspaces (prod-east1, future regions) without manual -var flags.
+  # The NS values come from `tofu output devservers_name_servers` in the child
+  # workspace once its hosted zone has been created.
+  prod_subdomain_delegations = {
+    prod = {
+      "east1.devservers.io" = [
+        "ns-1079.awsdns-06.org",
+        "ns-1999.awsdns-57.co.uk",
+        "ns-341.awsdns-42.com",
+        "ns-624.awsdns-14.net",
+      ]
+    }
+  }
+
   # Per-capacity-reservation AZ mappings (overrides gpu_subnet_assignments when CR is used)
   capacity_reservation_azs = {
     "prod-east1" = {

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/terraform-gpu-devservers/node-termination-handler.tf

@@ -12,7 +12,8 @@ resource "helm_release" "aws_node_termination_handler" {
   repository       = "https://aws.github.io/eks-charts"
   chart            = "aws-node-termination-handler"
   namespace        = "kube-system"
-  version          = "0.27.1"
+  # No version pin — chart versions advance frequently and my first guess (0.27.1)
+  # didn't exist. helm picks current latest stable. Add a pin once we hit a regression.
   cleanup_on_fail  = true
 
   values = [yamlencode({

{gpu_dev-0.5.26 → gpu_dev-0.5.28}/terraform-gpu-devservers/route53.tf

@@ -51,6 +51,19 @@ resource "aws_route53_record" "manual_subdomain_delegation" {
   records = var.subdomain_ns_records
 }
 
+# Auto-published NS delegations for child workspaces. Iterates prod_subdomain_delegations
+# (defined in main.tf) for the current workspace and creates an NS record per entry in
+# the parent zone — so `tofu apply` in prod automatically wires up east1.devservers.io
+# (and any future region) without -var flags.
+resource "aws_route53_record" "workspace_subdomain_delegations" {
+  for_each = local.effective_domain_name != "" && !local.is_subdomain ? try(local.prod_subdomain_delegations[terraform.workspace], {}) : {}
+  zone_id  = data.aws_route53_zone.parent[0].zone_id
+  name     = each.key
+  type     = "NS"
+  ttl      = 300
+  records  = each.value
+}
+
 # Use appropriate hosted zone (subdomain if created, otherwise parent)
 locals {
   hosted_zone_id = local.is_subdomain ? aws_route53_zone.subdomain[0].zone_id : (local.effective_domain_name != "" ? data.aws_route53_zone.parent[0].zone_id : "")