gpu-dev 0.5.1__tar.gz → 0.5.3__tar.gz

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.5.1
+Version: 0.5.3
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.5.1
+Version: 0.5.3
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py

@@ -498,9 +498,9 @@ def main(ctx: click.Context) -> None:
     "--gpu-type",
     "-t",
     type=click.Choice(
-        ["b200", "h200", "h100", "a100", "rtxpro6000", "a10g", "t4", "l4", "t4-small", "cpu-arm", "cpu-x86"], case_sensitive=False
+        ["b200", "h200", "h100", "h100-mig-1g", "h100-mig-2g", "h100-mig-3g", "a100", "rtxpro6000", "a10g", "t4", "l4", "t4-small", "cpu-arm", "cpu-x86"], case_sensitive=False
     ),
-    help="GPU type to reserve (b200/h200/h100/a100/rtxpro6000/a10g/t4/l4/t4-small/cpu-arm/cpu-x86)",
+    help="GPU type to reserve. Full GPUs: b200, h200, h100, a100, rtxpro6000, a10g, t4, l4, t4-small. H100 MIG slices (partial GPU on a single shared node): h100-mig-1g (10 GB / 1/7 H100 compute), h100-mig-2g (20 GB / 2/7 H100), h100-mig-3g (40 GB / 3/7 H100). CPU only: cpu-arm, cpu-x86.",
 )
 @click.option(
     "--hours",
@@ -656,6 +656,9 @@ def reserve(
             "t4-small": {"max_gpus": 1, "instance_type": "g4dn.xlarge"},
             "a100": {"max_gpus": 8, "instance_type": "p4d.24xlarge"},
             "h100": {"max_gpus": 8, "instance_type": "p5.48xlarge"},
+            "h100-mig-1g": {"max_gpus": 16, "instance_type": "p5.48xlarge"},
+            "h100-mig-2g": {"max_gpus": 8, "instance_type": "p5.48xlarge"},
+            "h100-mig-3g": {"max_gpus": 8, "instance_type": "p5.48xlarge"},
             "h200": {"max_gpus": 8, "instance_type": "p5e.48xlarge"},
             "b200": {"max_gpus": 8, "instance_type": "p6-b200.48xlarge"},
             "cpu-arm": {"max_gpus": 0, "instance_type": "c7g.4xlarge"},
@@ -724,11 +727,18 @@ def reserve(
                     return
 
                 max_gpus = gpu_configs[gpu_type_lower]["max_gpus"]
-                gpu_count = select_gpu_count_interactive(
-                    gpu_type_lower, max_gpus)
-                if gpu_count is None:
+                result = select_gpu_count_interactive(
+                    gpu_type_lower, max_gpus, availability_info=availability_info)
+                if result is None:
                     rprint("[yellow]Reservation cancelled.[/yellow]")
                     return
+                # If user picked a MIG slice, the function returns (gpu_type, count).
+                if isinstance(result, tuple):
+                    gpu_type, gpu_count = result
+                    gpu_type_lower = gpu_type.lower()
+                    max_gpus = gpu_configs[gpu_type_lower]["max_gpus"]
+                else:
+                    gpu_count = result
 
                 # Show distributed warning for interactive multinode selections (always show)
                 if gpu_count > max_gpus:
@@ -2399,6 +2409,9 @@ def _show_availability() -> None:
                 "a10g": "Ampere (sm80)",
                 "l4": "Ada Lovelace (sm89)",
                 "rtxpro6000": "Blackwell (sm120)",
+                "h100-mig-1g": "Hopper (sm90, MIG 10GB)",
+                "h100-mig-2g": "Hopper (sm90, MIG 20GB)",
+                "h100-mig-3g": "Hopper (sm90, MIG 40GB)",
                 "t4": "Turing (sm75)",
                 "cpu-x86": "CPU (x86_64)",
                 "cpu-arm": "CPU (arm64)",
@@ -2409,6 +2422,9 @@ def _show_availability() -> None:
                 "Blackwell (sm100)": 0,
                 "Blackwell (sm120)": 0,
                 "Hopper (sm90)": 1,
+                "Hopper (sm90, MIG 40GB)": 1,
+                "Hopper (sm90, MIG 20GB)": 1,
+                "Hopper (sm90, MIG 10GB)": 1,
                 "Ada Lovelace (sm89)": 2,
                 "Ampere (sm80)": 3,
                 "Turing (sm75)": 4,
@@ -2548,6 +2564,9 @@ def _show_availability_watch(interval: int) -> None:
                             "a10g": "Ampere (sm80)",
                             "l4": "Ada Lovelace (sm89)",
                             "rtxpro6000": "Blackwell (sm120)",
+                            "h100-mig-1g": "Hopper (sm90, MIG 10GB)",
+                            "h100-mig-2g": "Hopper (sm90, MIG 20GB)",
+                            "h100-mig-3g": "Hopper (sm90, MIG 40GB)",
                             "t4": "Turing (sm75)",
                             "cpu-x86": "CPU (x86_64)",
                             "cpu-arm": "CPU (arm64)",
@@ -2558,6 +2577,9 @@ def _show_availability_watch(interval: int) -> None:
                             "Blackwell (sm100)": 0,
                             "Blackwell (sm120)": 0,
                             "Hopper (sm90)": 1,
+                            "Hopper (sm90, MIG 40GB)": 1,
+                            "Hopper (sm90, MIG 20GB)": 1,
+                            "Hopper (sm90, MIG 10GB)": 1,
                             "Ada Lovelace (sm89)": 2,
                             "Ampere (sm80)": 3,
                             "Turing (sm75)": 4,

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/cli-tools/gpu-dev-cli/gpu_dev_cli/interactive.py

@@ -57,6 +57,13 @@ def select_gpu_type_interactive(
     if not check_interactive_support():
         return None
 
+    # Hide MIG slice SKUs from the top-level selector — reached via the h100 submenu.
+    # Direct `--gpu-type h100-mig-1g` still works for non-interactive scripts.
+    visible_info = {
+        gt: info for gt, info in availability_info.items()
+        if "-mig-" not in gt
+    }
+
     # Display availability table first
     console.print("\n[cyan]🖥️  GPU Availability:[/cyan]")
     table = Table()
@@ -67,7 +74,7 @@ def select_gpu_type_interactive(
     table.add_column("Est. Wait Time", style="magenta")
 
     choices = []
-    for gpu_type, info in availability_info.items():
+    for gpu_type, info in visible_info.items():
         available = info.get("available", 0)
         total = info.get("total", 0)
         queue_length = info.get("queue_length", 0)
@@ -143,8 +150,16 @@ def select_gpu_type_interactive(
         return None
 
 
-def select_gpu_count_interactive(gpu_type: str, max_gpus: int) -> Optional[int]:
-    """Interactive GPU count selection"""
+def select_gpu_count_interactive(
+    gpu_type: str,
+    max_gpus: int,
+    availability_info: Optional[Dict[str, Dict[str, Any]]] = None,
+):
+    """Interactive GPU count selection.
+
+    Returns int (gpu_count) for normal selections, or a (effective_gpu_type, gpu_count)
+    tuple when the user picks a MIG slice option from the h100 submenu.
+    """
     if not check_interactive_support():
         return None
 
@@ -157,6 +172,12 @@ def select_gpu_count_interactive(gpu_type: str, max_gpus: int) -> Optional[int]:
         valid_counts = [1, 2, 4]
         # Add multinode options
         multinode_counts = [8, 12, 16, 20, 24]  # multiples of 4
+    elif gpu_type == "h100-mig-1g":
+        valid_counts = [1, 2, 4, 8]
+        multinode_counts = []  # MIG slices live on a single node — no multinode
+    elif gpu_type in ["h100-mig-2g", "h100-mig-3g"]:
+        valid_counts = [1, 2, 4]
+        multinode_counts = []
     elif gpu_type == "g5g":
         valid_counts = [1, 2]
         multinode_counts = [4, 8]  # multiples of 4
@@ -168,6 +189,28 @@ def select_gpu_count_interactive(gpu_type: str, max_gpus: int) -> Optional[int]:
         # Add multinode options
         multinode_counts = [16, 24, 32, 40, 48]  # multiples of 8
 
+    # MIG slice submenu: only for h100. Each tuple is (target_gpu_type, gpu_count, gb_label).
+    mig_options = []
+    if gpu_type == "h100":
+        # Map to internal SKUs; the count menu surfaces 1/2/4 of each slice size.
+        mig_specs = [
+            ("h100-mig-1g", "10GB"),
+            ("h100-mig-2g", "20GB"),
+            ("h100-mig-3g", "40GB"),
+        ]
+        for sku, gb in mig_specs:
+            slice_max = {"h100-mig-1g": 16, "h100-mig-2g": 8, "h100-mig-3g": 8}[sku]
+            free = None
+            if availability_info and sku in availability_info:
+                free = availability_info[sku].get("available", 0)
+            for n in [1, 2, 4]:
+                if n > slice_max:
+                    continue
+                noun = "slice" if n == 1 else "slices"
+                avail_suffix = f"  [{free} free]" if free is not None else ""
+                label = f"{n} × {gb} {noun}{avail_suffix}"
+                mig_options.append((sku, n, label))
+
     # Filter single-node by actual max for this GPU type
     valid_counts = [count for count in valid_counts if count <= max_gpus]
 
@@ -177,7 +220,18 @@ def select_gpu_count_interactive(gpu_type: str, max_gpus: int) -> Optional[int]:
 
     choices = []
 
-    # Add single-node options
+    # MIG slice options come first (smallest unit), h100-only.
+    if mig_options:
+        choices.append(questionary.Separator(
+            "--- MIG slices (partial GPU, single node) ---"))
+        for sku, count, label in mig_options:
+            choices.append(questionary.Choice(title=label, value=(sku, count)))
+
+    # Full single-node options. Header only when slices were rendered above
+    # (otherwise the type already implies "Full GPUs").
+    if mig_options:
+        choices.append(questionary.Separator(
+            "--- Full GPUs (single node) ---"))
     for count in valid_counts:
         if count == 1:
             label = f"1 GPU (single node)"
@@ -185,7 +239,7 @@ def select_gpu_count_interactive(gpu_type: str, max_gpus: int) -> Optional[int]:
             label = f"{count} GPUs (single node)"
         choices.append(questionary.Choice(title=label, value=count))
 
-    # Add separator and multinode options
+    # Multinode at the bottom.
     if multinode_counts:
         choices.append(questionary.Separator(
             "--- Multinode (Distributed) ---"))

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/cli-tools/gpu-dev-cli/gpu_dev_cli/reservations.py

@@ -540,6 +540,9 @@ class ReservationManager:
                 "g5g": {"max_gpus": 2},
                 "a100": {"max_gpus": 8},
                 "h100": {"max_gpus": 8},
+                "h100-mig-1g": {"max_gpus": 16},
+                "h100-mig-2g": {"max_gpus": 8},
+                "h100-mig-3g": {"max_gpus": 8},
                 "h200": {"max_gpus": 8},
                 "b200": {"max_gpus": 8},
             }

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gpu-dev"
-version = "0.5.1"
+version = "0.5.3"
 description = "CLI tool for PyTorch GPU developer server reservations"
 authors = [{name = "PyTorch Team"}]
 readme = "cli-tools/gpu-dev-cli/README.md"

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/terraform-gpu-devservers/availability.tf

@@ -142,7 +142,7 @@ resource "aws_cloudwatch_event_rule" "asg_capacity_change" {
       "EC2 Instance Terminate Successful"
     ]
     detail = {
-      AutoScalingGroupName = [for gpu_type in keys(local.current_config.supported_gpu_types) : "${var.prefix}-gpu-nodes-${gpu_type}"]
+      AutoScalingGroupName = [for gpu_type, cfg in local.current_config.supported_gpu_types : "${var.prefix}-gpu-nodes-${gpu_type}" if !try(cfg.virtual, false)]
     }
   })
 

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/terraform-gpu-devservers/eks.tf

@@ -198,7 +198,7 @@ locals {
   # Flatten capacity reservations to create multiple ASGs when needed
   # Each CR entry must have a stable 'key' field so removing entries doesn't shift other ASG keys.
   gpu_capacity_reservations = flatten([
-    for gpu_type, gpu_config in local.current_config.supported_gpu_types : [
+    for gpu_type, gpu_config in local.current_config.supported_gpu_types : try(gpu_config.virtual, false) ? [] : [
       for cr_index, cr_config in try(local.capacity_reservations[terraform.workspace][gpu_type], [null]) : {
         gpu_type                = gpu_type
         gpu_config              = gpu_config
@@ -212,8 +212,13 @@ locals {
           ? lookup(local.capacity_reservation_azs[terraform.workspace], cr_config.id, local.gpu_subnet_assignments[terraform.workspace][gpu_type])
           : local.gpu_subnet_assignments[terraform.workspace][gpu_type]
         )
+        # Per-CR override for efa_network_cards (e.g. p5en.48xlarge caps at 16 vs p5e at 32)
+        efa_network_cards = cr_config != null ? try(cr_config.efa_network_cards, gpu_config.efa_network_cards) : gpu_config.efa_network_cards
+        # Optional MIG profile (e.g. "all-balanced", "all-1g.10gb"). When set, user-data labels the node so nvidia-mig-manager partitions the GPUs.
+        # Default to "" (not null) — null breaks templatefile() string interpolation downstream.
+        mig_profile = cr_config != null ? try(cr_config.mig_profile, "") : ""
         # Multi-EFA instances (>1 network card) must use private subnets (no public IP in launch template)
-        use_private_subnet = try(gpu_config.efa_network_cards, 0) > 1
+        use_private_subnet = (cr_config != null ? try(cr_config.efa_network_cards, try(gpu_config.efa_network_cards, 0)) : try(gpu_config.efa_network_cards, 0)) > 1
       }
     ]
   ])
@@ -363,7 +368,7 @@ resource "aws_launch_template" "gpu_dev_launch_template" {
       associate_public_ip_address = true
       security_groups             = [aws_security_group.gpu_dev_sg.id]
       subnet_id                   = each.value.gpu_config.use_placement_group ? null : local.public_subnet_map[each.value.subnet_az]
-      interface_type              = try(each.value.gpu_config.efa_network_cards, 0) > 0 ? "efa" : "interface"
+      interface_type              = try(each.value.efa_network_cards, 0) > 0 ? "efa" : "interface"
       delete_on_termination       = true
     }
   }
@@ -386,7 +391,7 @@ resource "aws_launch_template" "gpu_dev_launch_template" {
   # Each network card supports 2 device indices (0 and 1); device_index must be 0
   # since this is the only interface on each card
   dynamic "network_interfaces" {
-    for_each = each.value.use_private_subnet ? range(1, try(each.value.gpu_config.efa_network_cards, 1)) : []
+    for_each = each.value.use_private_subnet ? range(1, try(each.value.efa_network_cards, 1)) : []
     content {
       device_index          = 0
       interface_type        = "efa-only"
@@ -423,6 +428,7 @@ resource "aws_launch_template" "gpu_dev_launch_template" {
     region              = local.current_config.aws_region
     gpu_type            = local.gpu_type_kubernetes_labels[each.value.gpu_type]
     profiling_dedicated = try(each.value.gpu_config.profiling_dedicated, false)
+    mig_profile         = each.value.mig_profile != null ? each.value.mig_profile : ""
     container_image     = local.latest_image_uri
   }))
 

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/terraform-gpu-devservers/lambda/availability_updater/index.py

@@ -23,6 +23,13 @@ AVAILABILITY_TABLE = os.environ["AVAILABILITY_TABLE"]
 SUPPORTED_GPU_TYPES = json.loads(os.environ["SUPPORTED_GPU_TYPES"])
 
 
+def get_gpu_resource_name(gpu_type: str) -> str:
+    return SUPPORTED_GPU_TYPES.get(gpu_type, {}).get("k8s_resource", "nvidia.com/gpu")
+
+def get_node_label_value(gpu_type: str) -> str:
+    return SUPPORTED_GPU_TYPES.get(gpu_type, {}).get("node_gpu_type", gpu_type)
+
+
 def handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
     """Handle ASG capacity change events - update all GPU types"""
     try:
@@ -84,7 +91,9 @@ def update_gpu_availability(gpu_type: str, k8s_client=None) -> None:
         logger.info(f"Starting availability update for GPU type: {gpu_type}")
 
         # Get current ASG capacity - handle multiple ASGs per GPU type (e.g., capacity reservations)
-        asg_name_prefix = f"pytorch-gpu-dev-gpu-nodes-{gpu_type}"
+        # MIG SKUs share the underlying h100 ASGs (cr-dedicated MIG node), so use the physical type for ASG matching
+        asg_lookup_type = get_node_label_value(gpu_type)
+        asg_name_prefix = f"pytorch-gpu-dev-gpu-nodes-{asg_lookup_type}"
         logger.info(f"Checking ASGs matching pattern: {asg_name_prefix}*")
 
         # Get all ASGs and filter by name pattern
@@ -102,6 +111,9 @@ def update_gpu_availability(gpu_type: str, k8s_client=None) -> None:
         logger.info(f"Found {len(matching_asgs)} ASGs: {asg_names}")
 
         # Calculate total availability metrics across all matching ASGs
+        # For MIG SKUs we cannot tell from ASG alone which instances are MIG-partitioned;
+        # we override running_instances later from k8s allocatable.
+        is_mig_sku = "k8s_resource" in SUPPORTED_GPU_TYPES.get(gpu_type, {})
         desired_capacity = sum(asg["DesiredCapacity"] for asg in matching_asgs)
         running_instances = sum(
             len([
@@ -130,7 +142,7 @@ def update_gpu_availability(gpu_type: str, k8s_client=None) -> None:
                     logger.info(f"Checking CPU node availability for {gpu_type}")
                     # Count available slots by checking pod count on each node
                     v1 = client.CoreV1Api(k8s_client)
-                    nodes = v1.list_node(label_selector=f"GpuType={gpu_type}")
+                    nodes = v1.list_node(label_selector=f"GpuType={get_node_label_value(gpu_type)}")
 
                     total_available_slots = 0
                     for node in nodes.items:
@@ -178,16 +190,18 @@ def update_gpu_availability(gpu_type: str, k8s_client=None) -> None:
             try:
                 from kubernetes import client as k8s_client_lib
                 v1 = k8s_client_lib.CoreV1Api(k8s_client)
-                nodes = v1.list_node(label_selector=f"GpuType={gpu_type}")
+                node_label_value = get_node_label_value(gpu_type)
+                resource_name = get_gpu_resource_name(gpu_type)
+                nodes = v1.list_node(label_selector=f"GpuType={node_label_value}")
 
                 single_node_max = 0  # Max available on any single node
                 schedulable_total_gpus = 0  # Total GPUs on schedulable (non-cordoned) nodes
                 for node in nodes.items:
                     if is_node_ready_and_schedulable(node):
-                        available_on_node = get_available_gpus_on_node(v1, node)
+                        available_on_node = get_available_gpus_on_node(v1, node, gpu_type)
                         total_on_node = 0
                         if node.status.allocatable:
-                            gpu_allocatable = node.status.allocatable.get("nvidia.com/gpu", "0")
+                            gpu_allocatable = node.status.allocatable.get(resource_name, "0")
                             try:
                                 total_on_node = int(gpu_allocatable)
                             except (ValueError, TypeError):
@@ -203,6 +217,9 @@ def update_gpu_availability(gpu_type: str, k8s_client=None) -> None:
                             full_nodes_available += 1
 
                 total_gpus = schedulable_total_gpus
+                # For MIG SKUs override running_instances to the number of MIG-partitioned nodes
+                if is_mig_sku:
+                    running_instances = sum(1 for n in nodes.items if is_node_ready_and_schedulable(n) and int((n.status.allocatable or {}).get(resource_name, "0")) > 0)
 
                 # Calculate max reservable considering multinode scenarios
                 # Only high-end GPU types support multinode (up to 4 nodes = 32 GPUs)
@@ -276,7 +293,7 @@ def check_schedulable_gpus_for_type(k8s_client, gpu_type: str) -> int:
         logger.info(f"Created CoreV1Api client for {gpu_type}")
 
         # Get all nodes with the specified GPU type
-        gpu_type_selector = f"GpuType={gpu_type}"
+        gpu_type_selector = f"GpuType={get_node_label_value(gpu_type)}"
         logger.info(f"Querying nodes with label selector: {gpu_type_selector}")
 
         nodes = v1.list_node(label_selector=gpu_type_selector)
@@ -297,7 +314,7 @@ def check_schedulable_gpus_for_type(k8s_client, gpu_type: str) -> int:
 
             logger.info(f"Node {node.metadata.name} is ready, checking GPU availability")
             # Get available GPUs on this node
-            available_on_node = get_available_gpus_on_node(v1, node)
+            available_on_node = get_available_gpus_on_node(v1, node, gpu_type)
             total_schedulable += available_on_node
             logger.info(f"Node {node.metadata.name}: {available_on_node} GPUs available")
 
@@ -332,11 +349,12 @@ def is_node_ready_and_schedulable(node) -> bool:
         return False
 
 
-def get_available_gpus_on_node(v1_api, node) -> int:
-    """Get number of available GPUs on a specific node"""
+def get_available_gpus_on_node(v1_api, node, gpu_type: str = None) -> int:
+    """Get number of available GPUs (or MIG slices) on a specific node for the given SKU."""
     try:
         node_name = node.metadata.name
-        logger.info(f"Checking GPU availability on node: {node_name}")
+        resource_name = get_gpu_resource_name(gpu_type) if gpu_type else "nvidia.com/gpu"
+        logger.info(f"Checking GPU availability on node: {node_name} (resource={resource_name})")
 
         # Get all pods on this node
         logger.info(f"Querying pods on node {node_name}")
@@ -350,7 +368,7 @@ def get_available_gpus_on_node(v1_api, node) -> int:
                 for container in pod.spec.containers:
                     if container.resources and container.resources.requests:
                         gpu_request = container.resources.requests.get(
-                            "nvidia.com/gpu", "0"
+                            resource_name, "0"
                         )
                         try:
                             used_gpus += int(gpu_request)
@@ -360,7 +378,7 @@ def get_available_gpus_on_node(v1_api, node) -> int:
         # Get total GPUs on this node
         total_gpus = 0
         if node.status.allocatable:
-            gpu_allocatable = node.status.allocatable.get("nvidia.com/gpu", "0")
+            gpu_allocatable = node.status.allocatable.get(resource_name, "0")
             try:
                 total_gpus = int(gpu_allocatable)
             except (ValueError, TypeError):

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/terraform-gpu-devservers/lambda/reservation_processor/index.py

@@ -67,6 +67,10 @@ GPU_CONFIG = {
     "l4": {"instance_type": "g6.12xlarge", "max_gpus": 4, "cpus": 48, "memory_gb": 192, "efa_count": 1},
     "a10g": {"instance_type": "g5.12xlarge", "max_gpus": 4, "cpus": 48, "memory_gb": 192, "efa_count": 1},
     "rtxpro6000": {"instance_type": "g7e.24xlarge", "max_gpus": 4, "cpus": 96, "memory_gb": 1024, "efa_count": 2},
+    # MIG slices on a dedicated H100 node (all-balanced profile: per GPU = 2x1g.10gb + 1x2g.20gb + 1x3g.40gb)
+    "h100-mig-1g": {"instance_type": "p5.48xlarge", "max_gpus": 16, "cpus": 192, "memory_gb": 2048, "efa_count": 0, "k8s_resource": "nvidia.com/mig-1g.10gb", "node_gpu_type": "h100"},
+    "h100-mig-2g": {"instance_type": "p5.48xlarge", "max_gpus": 8, "cpus": 192, "memory_gb": 2048, "efa_count": 0, "k8s_resource": "nvidia.com/mig-2g.20gb", "node_gpu_type": "h100"},
+    "h100-mig-3g": {"instance_type": "p5.48xlarge", "max_gpus": 8, "cpus": 192, "memory_gb": 2048, "efa_count": 0, "k8s_resource": "nvidia.com/mig-3g.40gb", "node_gpu_type": "h100"},
     "t4-small": {"instance_type": "g4dn.2xlarge", "max_gpus": 1, "cpus": 8, "memory_gb": 32, "efa_count": 0},
     "g5g": {"instance_type": "g5g.2xlarge", "max_gpus": 2, "cpus": 8, "memory_gb": 32, "efa_count": 0},
     "a100": {"instance_type": "p4d.24xlarge", "max_gpus": 8, "cpus": 96, "memory_gb": 1152, "efa_count": 4},
@@ -78,6 +82,15 @@ GPU_CONFIG = {
 }
 GPU_CONFIG_DEFAULT = {"instance_type": "g4dn.12xlarge", "max_gpus": 4, "cpus": 48, "memory_gb": 192, "efa_count": 0}
 
+def get_gpu_resource_name(gpu_type: str) -> str:
+    """Kubernetes resource name for this SKU (nvidia.com/gpu or nvidia.com/mig-*)."""
+    return GPU_CONFIG.get(gpu_type, GPU_CONFIG_DEFAULT).get("k8s_resource", "nvidia.com/gpu")
+
+def get_node_gpu_type(gpu_type: str) -> str:
+    """Value of the GpuType node label to select. MIG SKUs map to their underlying physical type."""
+    return GPU_CONFIG.get(gpu_type, {}).get("node_gpu_type", gpu_type)
+
+
 # GPU types under maintenance - only whitelisted users can reserve
 # Set to {} to disable maintenance mode for all types
 GPU_MAINTENANCE = {}
@@ -232,7 +245,8 @@ def get_target_az_for_reservation(gpu_type, gpus_requested):
         # Get all nodes with the requested GPU type
         logger.info(
             f"Querying nodes for GPU type {gpu_type} with {gpus_requested} GPUs needed")
-        nodes = v1.list_node(label_selector=f"GpuType={gpu_type}")
+        node_label_value = get_node_gpu_type(gpu_type)
+        nodes = v1.list_node(label_selector=f"GpuType={node_label_value}")
 
         candidate_nodes = []
         all_ready_nodes = []
@@ -271,7 +285,7 @@ def get_target_az_for_reservation(gpu_type, gpus_requested):
                 continue
 
             # Check available GPU capacity on this node
-            available_gpus = get_available_gpus_on_node(v1, node)
+            available_gpus = get_available_gpus_on_node(v1, node, gpu_type)
 
             # Track all ready nodes (for fallback AZ when no single node has enough)
             all_ready_nodes.append({
@@ -2152,7 +2166,8 @@ def validate_reservation_request(request: dict[str, Any]) -> tuple[bool, str]:
 
     # Validate GPU type
     valid_gpu_types = ["t4", "l4", "a10g", "rtxpro6000", "t4-small", "a100",
-                       "h100", "h200", "b200", "cpu-arm", "cpu-x86"]
+                       "h100", "h100-mig-1g", "h100-mig-2g", "h100-mig-3g",
+                       "h200", "b200", "cpu-arm", "cpu-x86"]
     if gpu_type not in valid_gpu_types:
         error_msg = f"Invalid GPU type: {gpu_type}. Must be one of: {', '.join(valid_gpu_types)}"
         logger.error(error_msg)
@@ -2238,10 +2253,11 @@ def check_schedulable_gpus_for_type(k8s_client, gpu_type: str) -> int:
         nodes = v1.list_node()
         schedulable_gpus = 0
 
+        node_label_value = get_node_gpu_type(gpu_type)
         for node in nodes.items:
             # Check if node has the right GPU type label
             node_labels = node.metadata.labels or {}
-            if node_labels.get("GpuType") != gpu_type:
+            if node_labels.get("GpuType") != node_label_value:
                 continue
 
             # Check if node is ready and schedulable
@@ -2252,7 +2268,7 @@ def check_schedulable_gpus_for_type(k8s_client, gpu_type: str) -> int:
                 continue
 
             # Get available GPUs on this node
-            node_gpus = get_available_gpus_on_node(v1, node)
+            node_gpus = get_available_gpus_on_node(v1, node, gpu_type)
             schedulable_gpus += node_gpus
             logger.info(
                 f"Node {node.metadata.name}: {node_gpus} available {gpu_type.upper()} GPUs"
@@ -2278,13 +2294,14 @@ def check_max_gpus_on_single_node(gpu_type: str) -> int:
         nodes = v1.list_node()
         max_gpus = 0
 
+        node_label_value = get_node_gpu_type(gpu_type)
         for node in nodes.items:
             node_labels = node.metadata.labels or {}
-            if node_labels.get("GpuType") != gpu_type:
+            if node_labels.get("GpuType") != node_label_value:
                 continue
             if not is_node_ready_and_schedulable(node):
                 continue
-            node_gpus = get_available_gpus_on_node(v1, node)
+            node_gpus = get_available_gpus_on_node(v1, node, gpu_type)
             max_gpus = max(max_gpus, node_gpus)
 
         return max_gpus
@@ -2320,12 +2337,13 @@ def is_node_ready_and_schedulable(node) -> bool:
     return True
 
 
-def get_available_gpus_on_node(v1_api, node) -> int:
-    """Get the number of available GPUs on a specific node"""
+def get_available_gpus_on_node(v1_api, node, gpu_type: str = None) -> int:
+    """Get the number of available GPUs (or MIG slices) on a specific node for the given SKU."""
     try:
+        resource_name = get_gpu_resource_name(gpu_type) if gpu_type else "nvidia.com/gpu"
         # Get allocatable GPUs from node status
         allocatable = node.status.allocatable or {}
-        total_gpus = int(allocatable.get("nvidia.com/gpu", "0"))
+        total_gpus = int(allocatable.get(resource_name, "0"))
 
         if total_gpus == 0:
             return 0
@@ -2342,7 +2360,7 @@ def get_available_gpus_on_node(v1_api, node) -> int:
                     for container in pod.spec.containers:
                         if container.resources and container.resources.requests:
                             gpu_request = container.resources.requests.get(
-                                "nvidia.com/gpu", "0"
+                                resource_name, "0"
                             )
                             used_gpus += int(gpu_request)
 
@@ -2368,13 +2386,15 @@ def update_gpu_availability_table(
         total_gpus = 0
         running_instances = 0
 
+        node_label_value = get_node_gpu_type(gpu_type)
+        resource_name = get_gpu_resource_name(gpu_type)
         for node in nodes.items:
             node_labels = node.metadata.labels or {}
-            if node_labels.get("GpuType") == gpu_type:
+            if node_labels.get("GpuType") == node_label_value:
                 running_instances += 1
                 # Get allocatable GPUs from node status
                 allocatable = node.status.allocatable or {}
-                node_gpus = int(allocatable.get("nvidia.com/gpu", "0"))
+                node_gpus = int(allocatable.get(resource_name, "0"))
                 total_gpus += node_gpus
 
         # Get GPU configuration for this type (for gpus_per_instance)
@@ -2385,6 +2405,9 @@ def update_gpu_availability_table(
             "rtxpro6000": {"gpus_per_instance": 4},
             "a100": {"gpus_per_instance": 8},
             "h100": {"gpus_per_instance": 8},
+            "h100-mig-1g": {"gpus_per_instance": 16},
+            "h100-mig-2g": {"gpus_per_instance": 8},
+            "h100-mig-3g": {"gpus_per_instance": 8},
             "h200": {"gpus_per_instance": 8},
             "b200": {"gpus_per_instance": 8},
         }
@@ -3697,7 +3720,8 @@ def get_pod_resource_limits(gpu_count: int, gpu_type: str, is_multinode: bool =
     else:
         # GPU instances get proportional CPU/memory based on GPU allocation
         if gpu_count > 0:
-            limits["nvidia.com/gpu"] = str(gpu_count)
+            resource_name = config.get("k8s_resource", "nvidia.com/gpu")
+            limits[resource_name] = str(gpu_count)
 
             gpu_ratio = gpu_count / max_gpus if max_gpus > 0 else 1.0
 
@@ -3712,10 +3736,11 @@ def get_pod_resource_limits(gpu_count: int, gpu_type: str, is_multinode: bool =
                 "memory": f"{proportional_memory_limit}Gi"
             })
 
-    # EFA optimization: Only use EFA for full-node multinode deployments
+    # EFA optimization: Only use EFA for full-node multinode deployments (skip MIG slices)
     use_efa = (
         gpu_type != "t4-small" and
         not gpu_type.startswith("cpu-") and
+        "mig" not in gpu_type and
         is_multinode and
         gpu_count == max_gpus
     )
@@ -3742,7 +3767,8 @@ def get_pod_resource_requests(gpu_count: int, gpu_type: str, is_multinode: bool
         requests.update({"cpu": "2", "memory": "4Gi"})
     else:
         if gpu_count > 0:
-            requests["nvidia.com/gpu"] = str(gpu_count)
+            resource_name = config.get("k8s_resource", "nvidia.com/gpu")
+            requests[resource_name] = str(gpu_count)
             gpu_ratio = gpu_count / max_gpus if max_gpus > 0 else 1.0
 
             # Calculate proportional requests (reserve 10% for system overhead)
@@ -3756,10 +3782,11 @@ def get_pod_resource_requests(gpu_count: int, gpu_type: str, is_multinode: bool
                 "memory": f"{proportional_memory_request}Gi"
             })
 
-    # EFA: Only for full-node multinode deployments
+    # EFA: Only for full-node multinode deployments (skip MIG slices)
     use_efa = (
         gpu_type != "t4-small" and
         not gpu_type.startswith("cpu-") and
+        "mig" not in gpu_type and
         is_multinode and
         gpu_count == max_gpus
     )
@@ -5243,7 +5270,7 @@ EOF
                 )
             ] if _pod_uses_efa(gpu_count, gpu_type, is_multinode) else []),
             node_selector={
-                "GpuType": gpu_type,
+                "GpuType": get_node_gpu_type(gpu_type),
                 **({} if target_az is None else {"topology.kubernetes.io/zone": target_az})
             },
             # Node affinity for profiling-dedicated preference
@@ -6846,7 +6873,7 @@ def update_pod_status_and_events(k8s_client, pod_name: str, reservation_id: str)
                                             f"Failed to convert to queued: {queue_err}")
 
                         # Show user-friendly scheduling messages while waiting
-                        if "Insufficient nvidia.com/gpu" in event.message:
+                        if "Insufficient nvidia.com/" in event.message and "gpu" in event.message.lower():
                             # Check if it's a fragmentation issue (GPUs exist but not enough on single node)
                             try:
                                 reservations_table = dynamodb.Table(
@@ -6882,7 +6909,7 @@ def update_pod_status_and_events(k8s_client, pod_name: str, reservation_id: str)
                                 k8s_client_temp = get_k8s_client()
                                 v1 = client.CoreV1Api(k8s_client_temp)
                                 nodes = v1.list_node(
-                                    label_selector=f"GpuType={gpu_type}")
+                                    label_selector=f"GpuType={get_node_gpu_type(gpu_type)}")
 
                                 if len(nodes.items) == 0:
                                     # No nodes exist for this GPU type - fail immediately

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/terraform-gpu-devservers/lambda.tf

@@ -180,8 +180,8 @@ resource "aws_lambda_function" "reservation_processor" {
       HOSTED_ZONE_ID                     = local.effective_domain_name != "" ? local.hosted_zone_id : ""
       SSH_DOMAIN_MAPPINGS_TABLE          = local.effective_domain_name != "" ? aws_dynamodb_table.ssh_domain_mappings.name : ""
       SSL_CERTIFICATE_ARN                = local.effective_domain_name != "" ? aws_acm_certificate.wildcard[0].arn : ""
-      LAMBDA_VERSION                     = "0.5.1"
-      MIN_CLI_VERSION                    = "0.5.1"
+      LAMBDA_VERSION                     = "0.5.3"
+      MIN_CLI_VERSION                    = "0.5.2"
       DISK_CONTENTS_BUCKET               = aws_s3_bucket.disk_contents.bucket
       OPERATIONS_TABLE                   = aws_dynamodb_table.operations.name
     }, local.alb_env_vars)

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/terraform-gpu-devservers/main.tf

@@ -152,7 +152,7 @@ locals {
           efa_network_cards   = 8 # p6-b200.48xlarge supports max 8 network cards
         }
         "h200" = {
-          instance_type       = "p5e.48xlarge" # Match capacity reservation type
+          instance_type       = "p5en.48xlarge" # Match capacity reservation type
           instance_types      = ["p5e.48xlarge", "p5en.48xlarge"]
           instance_count      = 4 # Fallback default (not used when capacity_reservations defined)
           gpus_per_instance   = 8
@@ -216,6 +216,45 @@ locals {
           architecture        = "x86_64"
           efa_network_cards   = 2
         }
+        # MIG slice SKUs — virtual: do NOT create an ASG. Surfaces the SKU to availability_updater
+        # + reservation_processor. Backed by the H100 CR labelled with mig_profile=all-balanced
+        # (per GPU = 2x1g.10gb + 1x2g.20gb + 1x3g.40gb).
+        "h100-mig-1g" = {
+          instance_type       = null
+          instance_types      = null
+          instance_count      = 0
+          gpus_per_instance   = 16 # 8 GPUs * 2 slices/GPU
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 0
+          virtual             = true
+          k8s_resource        = "nvidia.com/mig-1g.10gb"
+          node_gpu_type       = "h100"
+        }
+        "h100-mig-2g" = {
+          instance_type       = null
+          instance_types      = null
+          instance_count      = 0
+          gpus_per_instance   = 8 # 8 GPUs * 1 slice/GPU
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 0
+          virtual             = true
+          k8s_resource        = "nvidia.com/mig-2g.20gb"
+          node_gpu_type       = "h100"
+        }
+        "h100-mig-3g" = {
+          instance_type       = null
+          instance_types      = null
+          instance_count      = 0
+          gpus_per_instance   = 8 # 8 GPUs * 1 slice/GPU
+          use_placement_group = false
+          architecture        = "x86_64"
+          efa_network_cards   = 0
+          virtual             = true
+          k8s_resource        = "nvidia.com/mig-3g.40gb"
+          node_gpu_type       = "h100"
+        }
         "cpu-arm" = {
           instance_type       = "c7g.8xlarge"
           instance_types      = null
@@ -267,11 +306,13 @@ locals {
         { key = "cr0", id = "cr-0a3f49b96fe03ca04", instance_count = 4 }, # H100 reservation us-east-2c (p5.48xlarge)
         { key = "cr1", id = null, instance_count = 2 },                   # H100 on-demand (2 instances)
         { key = "cr2", id = "cr-044bc72b0a6b56062", instance_count = 4 }, # H100 reservation us-east-2a (4 instances)
+        { key = "cr3", id = "cr-0211ea1e8d3a3c79e", instance_count = 1, mig_profile = "all-balanced" }, # H100 reservation us-east-2c (1 instance, MIG-dedicated, all-balanced: 2x1g.10gb + 1x2g.20gb + 1x3g.40gb per GPU)
       ]
       h200 = [
         { key = "cr0", id = "cr-0f6d0766f5d3339e6", instance_count = 2 }, # H200 capacity block (may be expired - keep to prevent ASG destroy)
         { key = "cr1", id = "cr-06c9c978dea756a26", instance_count = 3 }, # H200 reservation (3 instances)
         { key = "cr2", id = null, instance_count = 2 },                   # H200 on-demand (2 instances)
+        { key = "cr3", id = "cr-02949f61f1a761b54", instance_count = 1, efa_network_cards = 16 }, # H200 reservation us-east-2a (1 instance, 8 GPUs, p5en.48xlarge max 16 EFA)
       ]
       b200 = [
         { key = "cr0", id = "cr-0c366fb8339a10f69", instance_count = 0 }, # B200 reservation us-east-2a (disabled - CR freed)
@@ -323,9 +364,11 @@ locals {
       # H200 capacity reservations
       "cr-0f6d0766f5d3339e6" = "tertiary" # us-east-2c (may be expired - kept to prevent ASG destroy)
       "cr-06c9c978dea756a26" = "tertiary"  # us-east-2c
+      "cr-02949f61f1a761b54" = "primary"   # us-east-2a
       # H100 capacity reservations
       "cr-0a3f49b96fe03ca04" = "tertiary" # us-east-2c (p5.48xlarge)
       "cr-044bc72b0a6b56062" = "primary"  # us-east-2a (p5.48xlarge)
+      "cr-0211ea1e8d3a3c79e" = "tertiary" # us-east-2c (p5.48xlarge, MIG-dedicated)
       # A100 capacity reservation
       "cr-01cc0f00f28b095af" = "primary" # us-east-2a
     }

{gpu_dev-0.5.1 → gpu_dev-0.5.3}/terraform-gpu-devservers/templates/al2023-user-data.sh

@@ -136,7 +136,7 @@ spec:
         cpu: "2"
         memory: "4Gi"
     flags:
-      - --node-labels=NodeType=gpu,GpuType=${gpu_type},nvidia.com/gpu.deploy.driver=false${profiling_dedicated ? ",gpu.monitoring/profiling-dedicated=true,nvidia.com/gpu.deploy.dcgm-exporter=false" : ""}
+      - --node-labels=NodeType=gpu,GpuType=${gpu_type},nvidia.com/gpu.deploy.driver=false${profiling_dedicated ? ",gpu.monitoring/profiling-dedicated=true,nvidia.com/gpu.deploy.dcgm-exporter=false" : ""}${mig_profile != "" ? ",nvidia.com/mig.config=${mig_profile}" : ""}
 EOF
 
 # Configure EFA if hardware present (BEFORE nodeadm so kubelet sees hugepages)