gpu-dev 0.5.12__tar.gz → 0.5.13__tar.gz

{gpu_dev-0.5.12 → gpu_dev-0.5.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.5.12
+Version: 0.5.13
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.5.12 → gpu_dev-0.5.13}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.5.12
+Version: 0.5.13
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.5.12 → gpu_dev-0.5.13}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/SOURCES.txt

@@ -47,6 +47,8 @@ terraform-gpu-devservers/git-cache.tf
 terraform-gpu-devservers/kubernetes.tf
 terraform-gpu-devservers/lambda.tf
 terraform-gpu-devservers/main.tf
+terraform-gpu-devservers/mig-config.tf
+terraform-gpu-devservers/mig-parted-config.yaml
 terraform-gpu-devservers/monitoring.tf
 terraform-gpu-devservers/outputs.tf
 terraform-gpu-devservers/pyproject.toml
@@ -101,7 +103,6 @@ terraform-gpu-devservers/migrations/check_snapshots.py
 terraform-gpu-devservers/migrations/migrate_disks_to_named.py
 terraform-gpu-devservers/migrations/run_backfill.sh
 terraform-gpu-devservers/scripts/CLEANUP_GUIDE.md
-terraform-gpu-devservers/scripts/b200-mig-setup.sh
 terraform-gpu-devservers/scripts/detect_empty_volumes.sh
 terraform-gpu-devservers/scripts/ec2_avail_probe.sh
 terraform-gpu-devservers/scripts/inspect_user_data.sh

{gpu_dev-0.5.12 → gpu_dev-0.5.13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gpu-dev"
-version = "0.5.12"
+version = "0.5.13"
 description = "CLI tool for PyTorch GPU developer server reservations"
 authors = [{name = "PyTorch Team"}]
 readme = "cli-tools/gpu-dev-cli/README.md"

{gpu_dev-0.5.12 → gpu_dev-0.5.13}/terraform-gpu-devservers/kubernetes.tf

@@ -305,6 +305,14 @@ resource "helm_release" "nvidia_gpu_operator" {
     value = "all-disabled"
   }
 
+  # Read profiles from our forked ConfigMap (managed in mig-config.tf) instead of the
+  # operator's auto-created default-mig-parted-config. Lets us add custom mixed profiles
+  # like b200-6full-2mig-balanced without ClusterPolicy reconciliation reverting our edits.
+  set {
+    name  = "migManager.config.name"
+    value = "gpu-dev-mig-parted-config"
+  }
+
   set {
     name  = "nodeStatusExporter.enabled"
     value = "true"

{gpu_dev-0.5.12 → gpu_dev-0.5.13}/terraform-gpu-devservers/lambda.tf

@@ -180,7 +180,7 @@ resource "aws_lambda_function" "reservation_processor" {
       HOSTED_ZONE_ID                     = local.effective_domain_name != "" ? local.hosted_zone_id : ""
       SSH_DOMAIN_MAPPINGS_TABLE          = local.effective_domain_name != "" ? aws_dynamodb_table.ssh_domain_mappings.name : ""
       SSL_CERTIFICATE_ARN                = local.effective_domain_name != "" ? aws_acm_certificate.wildcard[0].arn : ""
-      LAMBDA_VERSION                     = "0.5.12"
+      LAMBDA_VERSION                     = "0.5.13"
       MIN_CLI_VERSION                    = "0.5.9"
       DISK_CONTENTS_BUCKET               = aws_s3_bucket.disk_contents.bucket
       OPERATIONS_TABLE                   = aws_dynamodb_table.operations.name

gpu_dev-0.5.13/terraform-gpu-devservers/mig-config.tf

@@ -0,0 +1,55 @@
+# mig-config.tf — fork the NVIDIA mig-parted-config ConfigMap so we can add custom profiles
+# without fighting NVIDIA ClusterPolicy's reconciliation of its default-mig-parted-config.
+#
+# The vendored mig-parted-config.yaml in this directory mirrors the upstream profiles plus our
+# additions (e.g. b200-6full-2mig-balanced). Helm is told to use this ConfigMap by name via
+# migManager.config.name in kubernetes.tf, so the GPU operator skips creating its default and
+# reads ours instead.
+
+resource "kubernetes_config_map" "gpu_dev_mig_parted_config" {
+  metadata {
+    name      = "gpu-dev-mig-parted-config"
+    namespace = "gpu-operator"
+    labels = {
+      "app.kubernetes.io/managed-by" = "terraform"
+      "app.kubernetes.io/part-of"    = "gpu-dev-servers"
+    }
+  }
+
+  data = {
+    "config.yaml" = file("${path.module}/mig-parted-config.yaml")
+  }
+
+  # The gpu-operator namespace is created by the helm release; depend on that so this ConfigMap
+  # lands AFTER the namespace exists.
+  depends_on = [helm_release.nvidia_gpu_operator]
+}
+
+# Optional declarative B200 MIG node label. Set b200_mig_node_name in tfvars (or override the
+# variable's default below) to dedicate a specific B200 node to the mixed profile. Empty string
+# means "no node currently labelled" — the existing all-disabled stays in effect.
+variable "b200_mig_node_name" {
+  description = "Hostname of the B200 node to label with nvidia.com/mig.config=b200-6full-2mig-balanced. Leave empty to skip."
+  type        = string
+  default     = ""
+}
+
+resource "kubernetes_labels" "b200_mig_node" {
+  count = var.b200_mig_node_name == "" ? 0 : 1
+
+  api_version = "v1"
+  kind        = "Node"
+
+  metadata {
+    name = var.b200_mig_node_name
+  }
+
+  labels = {
+    "nvidia.com/mig.config" = "b200-6full-2mig-balanced"
+  }
+
+  # Take ownership of the label even if another tool (kubectl, gpu-operator) set it.
+  force = true
+
+  depends_on = [kubernetes_config_map.gpu_dev_mig_parted_config]
+}

gpu_dev-0.5.13/terraform-gpu-devservers/mig-parted-config.yaml

@@ -0,0 +1,528 @@
+version: v1
+mig-configs:
+  all-disabled:
+    - devices: all
+      mig-enabled: false
+
+  all-enabled:
+    - devices: all
+      mig-enabled: true
+      mig-devices: {}
+
+  # A100-40GB, A800-40GB
+  all-1g.5gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.5gb": 7
+
+  all-1g.5gb.me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.5gb+me": 1
+
+  all-2g.10gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.10gb": 3
+
+  all-3g.20gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "3g.20gb": 2
+
+  all-4g.20gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.20gb": 1
+
+  all-7g.40gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "7g.40gb": 1
+
+  # RTX-PRO-6000-96GB
+  all-1g.24gb.gfx:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.24gb+gfx": 4
+
+  all-1g.24gb.me.all:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.24gb+me.all": 1
+  
+  all-1g.24gb-me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.24gb-me": 4
+
+  all-2g.48gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.48gb": 2
+
+  all-2g.48gb.gfx:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.48gb+gfx": 2
+
+  all-2g.48gb.me.all:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.48gb+me.all": 1
+
+  all-2g.48gb-me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.48gb-me": 2
+
+  all-4g.96gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.96gb": 1
+
+  all-4g.96gb.gfx:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.96gb+gfx": 1
+
+  # H100-80GB, H800-80GB, A100-80GB, A800-80GB, A100-40GB, A800-40GB
+  all-1g.10gb:
+    # H100-80GB, H800-80GB, A100-80GB, A800-80GB
+    - device-filter: ["0x233010DE", "0x233110DE", "0x232210DE", "0x20B210DE", "0x20B510DE", "0x20F310DE", "0x20F510DE", "0x232410DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.10gb": 7
+
+    # A100-40GB, A800-40GB
+    - device-filter: ["0x20B010DE", "0x20B110DE", "0x20F110DE", "0x20F610DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.10gb": 4
+
+  # H100-80GB, H800-80GB, A100-80GB, A800-80GB
+  all-1g.10gb.me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.10gb+me": 1
+
+  # H100-80GB, H800-80GB, A100-80GB, A800-80GB
+  all-1g.20gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.20gb": 4
+
+  # GB200, B200
+  all-1g.23gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.23gb": 7
+
+  # GB200, B200
+  all-1g.23gb.me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.23gb+me": 1
+
+  all-1g.24gb.me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.24gb+me": 1
+
+  all-2g.20gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.20gb": 3
+
+  all-3g.40gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "3g.40gb": 2
+
+  all-4g.40gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.40gb": 1
+
+  all-7g.80gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "7g.80gb": 1
+
+  # A30-24GB
+  all-1g.6gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.6gb": 4
+
+  all-1g.6gb.me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.6gb+me": 1
+
+  all-2g.12gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.12gb": 2
+
+  all-2g.12gb.me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.12gb+me": 1
+
+  all-4g.24gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.24gb": 1
+
+  # H100 NVL, H800 NVL, GH200
+  all-1g.12gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.12gb": 7
+
+  all-1g.12gb.me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.12gb+me": 1
+
+  all-1g.24gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.24gb": 4
+
+  all-1g.45gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.45gb": 4
+
+  all-1g.47gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.47gb": 4
+
+  all-2g.24gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.24gb": 3
+
+  all-2g.45gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.45gb": 3
+
+  all-2g.47gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.47gb": 3
+
+  # H100 NVL, H800 NVL
+  all-3g.47gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "3g.47gb": 2
+
+  all-4g.47gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.47gb": 1
+
+  all-7g.94gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "7g.94gb": 1
+
+  # H100-96GB, PG506-96GB, GH200
+  all-3g.48gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "3g.48gb": 2
+
+  all-3g.90gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "3g.90gb": 2
+
+  all-3g.93gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "3g.93gb": 2
+
+  all-3g.95gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "3g.95gb": 2
+
+  all-4g.48gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.48gb": 1
+
+  all-4g.90gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.90gb": 1
+
+  all-4g.93gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.93gb": 1
+
+  all-4g.95gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.95gb": 1
+
+  all-7g.96gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "7g.96gb": 1
+
+  all-7g.180gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "7g.180gb": 1
+
+  all-7g.186gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "7g.186gb": 1
+
+  all-7g.189gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "7g.189gb": 1
+
+  # GB200 HGX, B200, GH200 144G HBM3e, H200-141GB, H200 NVL, H100-96GB, GH200, H100 NVL, H800 NVL, H100-80GB, H800-80GB, A800-40GB, A800-80GB, A100-40GB, A100-80GB, A30-24GB, PG506-96GB
+  all-balanced:
+    # GB200 HGX
+    - device-filter: ["0x294110DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.23gb": 2
+        "2g.47gb": 1
+        "3g.93gb": 1
+    
+    # RTX-PRO-6000-96GB
+    - device-filter: ["0x2BB510DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.24gb": 2
+        "2g.48gb": 1
+
+    # B200
+    - device-filter: ["0x290110DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.23gb": 2
+        "2g.45gb": 1
+        "3g.90gb": 1
+
+    # GH200 144G HBM3e
+    - device-filter: ["0x234810DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.18gb": 2
+        "2g.36gb": 1
+        "3g.72gb": 1
+
+    # H200 141GB, H200 NVL
+    - device-filter: ["0x233510DE", "0x233B10DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.18gb": 2
+        "2g.35gb": 1
+        "3g.71gb": 1
+
+    # H100 NVL, H800 NVL
+    - device-filter: ["0x232110DE", "0x233A10DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.12gb": 2
+        "2g.24gb": 1
+        "3g.47gb": 1
+
+    # H100-80GB, H800-80GB, A100-80GB, A800-80GB
+    - device-filter: ["0x233010DE", "0x233110DE", "0x232210DE", "0x20B210DE", "0x20B510DE", "0x20F310DE", "0x20F510DE", "0x232410DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.10gb": 2
+        "2g.20gb": 1
+        "3g.40gb": 1
+
+    # A100-40GB, A800-40GB
+    - device-filter: ["0x20B010DE", "0x20B110DE", "0x20F110DE", "0x20F610DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.5gb": 2
+        "2g.10gb": 1
+        "3g.20gb": 1
+
+    # A30-24GB
+    - device-filter: "0x20B710DE"
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.6gb": 2
+        "2g.12gb": 1
+
+    # H100-96GB, PG506-96GB, GH200, H20
+    - device-filter: ["0x234210DE", "0x233D10DE", "0x20B610DE", "0x232910DE"]
+      devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.12gb": 2
+        "2g.24gb": 1
+        "3g.48gb": 1
+
+  # H200-141GB, GH200 144G HBM3e
+  all-1g.18gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.18gb": 7
+
+  all-1g.18gb.me:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.18gb+me": 1
+
+  # H200-141GB
+  all-1g.35gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.35gb": 4
+
+  all-2g.35gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.35gb": 3
+
+  all-3g.71gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "3g.71gb": 2
+
+  all-4g.71gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.71gb": 1
+
+  all-7g.141gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "7g.141gb": 1
+
+  # GH200 144G HBM3e
+  all-1g.36gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "1g.36gb": 4
+
+  all-2g.36gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "2g.36gb": 3
+
+  all-3g.72gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "3g.72gb": 2
+
+  all-4g.72gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "4g.72gb": 1
+
+  all-7g.144gb:
+    - devices: all
+      mig-enabled: true
+      mig-devices:
+        "7g.144gb": 1
+
+  # Custom: B200 mixed split — GPUs 0-5 stay full (reservable as --gpu-type b200),
+  # GPUs 6-7 partitioned per-GPU into 2x1g.23gb + 1x2g.45gb + 1x3g.90gb.
+  # Per node: 6 full + 4 small + 2 medium + 2 large slices.
+  b200-6full-2mig-balanced:
+    - device-filter: ["0x290110DE"]
+      devices: [0, 1, 2, 3, 4, 5]
+      mig-enabled: false
+    - device-filter: ["0x290110DE"]
+      devices: [6, 7]
+      mig-enabled: true
+      mig-devices:
+        "1g.23gb": 2
+        "2g.45gb": 1
+        "3g.90gb": 1

gpu_dev-0.5.12/terraform-gpu-devservers/scripts/b200-mig-setup.sh

@@ -1,75 +0,0 @@
-#!/bin/bash
-# Post-deploy setup for B200 MIG split (6 full + 2 partitioned per node).
-# Run ONCE after PR #77 is merged + tf applied + the new docker/lambda is live.
-
-set -e
-
-NS=gpu-operator
-CM=default-mig-parted-config
-PROFILE_NAME=b200-6full-2mig-balanced
-
-echo "=== Checking current MIG profile in ConfigMap ==="
-if kubectl -n "$NS" get configmap "$CM" -o jsonpath='{.data.config\.yaml}' | grep -q "$PROFILE_NAME:"; then
-    echo "Profile $PROFILE_NAME already present — skipping ConfigMap edit"
-else
-    echo "Profile $PROFILE_NAME missing. Patching ConfigMap..."
-
-    # Save current ConfigMap content
-    kubectl -n "$NS" get configmap "$CM" -o yaml > /tmp/mig-config-backup.yaml
-    echo "Backup saved to /tmp/mig-config-backup.yaml"
-
-    # Append our profile under mig-configs:
-    # NOTE: this is a sed-driven append. ClusterPolicy's controller MAY revert this if it
-    # reconciles. If you see the profile disappear, re-run this script. If it keeps reverting,
-    # we'll need to fork the ConfigMap (next iteration).
-    kubectl -n "$NS" get configmap "$CM" -o jsonpath='{.data.config\.yaml}' > /tmp/mig-config.yaml
-
-    cat >> /tmp/mig-config.yaml <<'EOF'
-
-  # Mixed B200 split: GPUs 0-5 stay full (reservable as --gpu-type b200), GPUs 6-7 partitioned.
-  # Per partitioned GPU: 2x 1g.23gb + 1x 2g.45gb + 1x 3g.90gb. Per node: 6 full + 4 small + 2 medium + 2 large.
-  b200-6full-2mig-balanced:
-    - device-filter: ["0x290110DE"]
-      devices: [0, 1, 2, 3, 4, 5]
-      mig-enabled: false
-    - device-filter: ["0x290110DE"]
-      devices: [6, 7]
-      mig-enabled: true
-      mig-devices:
-        "1g.23gb": 2
-        "2g.45gb": 1
-        "3g.90gb": 1
-EOF
-
-    # Re-encode and patch
-    kubectl -n "$NS" create configmap "$CM" --from-file=config.yaml=/tmp/mig-config.yaml --dry-run=client -o yaml \
-        | kubectl -n "$NS" patch configmap "$CM" --patch-file=/dev/stdin
-    echo "ConfigMap patched."
-fi
-
-echo
-echo "=== Picking a B200 node to label ==="
-NODE=$(kubectl get nodes -l GpuType=b200 -o jsonpath='{.items[0].metadata.name}')
-if [ -z "$NODE" ]; then
-    echo "No B200 nodes found. Exiting."
-    exit 1
-fi
-echo "Will label: $NODE"
-read -p "Proceed? (y/N): " CONFIRM
-if [ "$CONFIRM" != "y" ]; then
-    echo "Aborted."
-    exit 0
-fi
-
-kubectl label node "$NODE" "nvidia.com/mig.config=$PROFILE_NAME" --overwrite
-echo "Node labelled. nvidia-mig-manager will partition GPUs 6-7 (drains existing pods if any)."
-echo
-echo "Watch progress with:"
-echo "  kubectl logs -n gpu-operator -l app=nvidia-mig-manager -f"
-echo "  kubectl get node $NODE -o jsonpath='{.status.allocatable}' | jq ."
-echo
-echo "After ~2-5 min, allocatable should show:"
-echo "  nvidia.com/gpu:           6"
-echo "  nvidia.com/mig-1g.23gb:   4"
-echo "  nvidia.com/mig-2g.45gb:   2"
-echo "  nvidia.com/mig-3g.90gb:   2"