gpu-dev 0.6.5__tar.gz → 0.6.6__tar.gz

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.6.5
+Version: 0.6.6
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.6.5
+Version: 0.6.6
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ sdk/python/README.md
 sdk/python/pyproject.toml
 sdk/python/examples/batch_multi_gpu.py
 sdk/python/examples/interactive_debug.py
+sdk/python/examples/parallel_experiments.ipynb
 sdk/python/examples/quickstart.ipynb
 sdk/python/examples/run_tests.py
 sdk/python/examples/submit_job.py

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/cli-tools/gpu-dev-cli/gpu_dev_cli/disks.py

@@ -243,6 +243,13 @@ def list_disks(user_id: str, config: Config) -> List[Dict]:
     except Exception:
         pass
 
+    # Filter out expired deleted disks (delete_date has passed)
+    today = datetime.now(timezone.utc).strftime('%Y-%m-%d')
+    disks = [
+        d for d in disks
+        if not (d.get('is_deleted') and d.get('delete_date') and str(d['delete_date']) <= today)
+    ]
+
     # Sort by last_used (most recent first)
     disks.sort(key=lambda d: d['last_used'] or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
 

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gpu-dev"
-version = "0.6.5"
+version = "0.6.6"
 description = "CLI tool for PyTorch GPU developer server reservations"
 authors = [{name = "PyTorch Team"}]
 readme = "cli-tools/gpu-dev-cli/README.md"

gpu_dev-0.6.6/sdk/python/examples/parallel_experiments.ipynb

@@ -0,0 +1,360 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Parallel Experiments with Persistent Disks\n",
+    "\n",
+    "This notebook demonstrates:\n",
+    "1. Creating a base environment on a persistent disk\n",
+    "2. Making changes (installing packages, modifying code)\n",
+    "3. Cloning the disk for parallel experiments\n",
+    "4. Running two experiments simultaneously on different GPUs\n",
+    "5. Comparing results and measuring timings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install -e .. -q"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
+    "from gpu_dev import GpuDev\n",
+    "\n",
+    "client = GpuDev()\n",
+    "print(f\"SDK v{__import__('gpu_dev').__version__}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 1: Create Base Environment on Persistent Disk\n",
+    "\n",
+    "Reserve a GPU with a persistent disk and set up the base experiment."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "t0 = time.time()\n",
+    "\n",
+    "base = client.reserve(\n",
+    "    gpu_type=\"t4\",\n",
+    "    gpu_count=1,\n",
+    "    hours=1,\n",
+    "    disk_name=\"experiment-base\",\n",
+    "    name=\"base-setup\",\n",
+    ")\n",
+    "\n",
+    "reserve_time = time.time() - t0\n",
+    "print(f\"Reserved in {reserve_time:.1f}s\")\n",
+    "print(f\"Disk: {base.disk_name}\")\n",
+    "print(f\"GPU:  {base.gpu_type} x{base.gpu_count}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Set up the base experiment: install packages + write training script\n",
+    "base.exec(\"pip install -q wandb timm\")\n",
+    "\n",
+    "# Write a parameterized training script\n",
+    "base.exec(r\"\"\"\n",
+    "cat > /home/dev/train.py << 'SCRIPT'\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import time\n",
+    "import json\n",
+    "import os\n",
+    "import sys\n",
+    "\n",
+    "# Read experiment config from env\n",
+    "LR = float(os.environ.get('LR', '0.001'))\n",
+    "BATCH_SIZE = int(os.environ.get('BATCH_SIZE', '64'))\n",
+    "EPOCHS = int(os.environ.get('EPOCHS', '5'))\n",
+    "EXP_NAME = os.environ.get('EXP_NAME', 'default')\n",
+    "\n",
+    "print(f\"Experiment: {EXP_NAME}\")\n",
+    "print(f\"Config: lr={LR}, batch_size={BATCH_SIZE}, epochs={EPOCHS}\")\n",
+    "print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n",
+    "print(f\"PyTorch: {torch.__version__}\")\n",
+    "\n",
+    "# Simple CNN on synthetic data\n",
+    "model = nn.Sequential(\n",
+    "    nn.Conv2d(3, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),\n",
+    "    nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),\n",
+    "    nn.Flatten(), nn.Linear(64 * 8 * 8, 10)\n",
+    ").cuda()\n",
+    "\n",
+    "optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n",
+    "criterion = nn.CrossEntropyLoss()\n",
+    "\n",
+    "results = {'experiment': EXP_NAME, 'lr': LR, 'batch_size': BATCH_SIZE, 'losses': [], 'epoch_times': []}\n",
+    "\n",
+    "for epoch in range(EPOCHS):\n",
+    "    t_start = time.time()\n",
+    "    epoch_loss = 0\n",
+    "    for step in range(50):\n",
+    "        x = torch.randn(BATCH_SIZE, 3, 32, 32, device='cuda')\n",
+    "        y = torch.randint(0, 10, (BATCH_SIZE,), device='cuda')\n",
+    "        loss = criterion(model(x), y)\n",
+    "        optimizer.zero_grad()\n",
+    "        loss.backward()\n",
+    "        optimizer.step()\n",
+    "        epoch_loss += loss.item()\n",
+    "    avg_loss = epoch_loss / 50\n",
+    "    epoch_time = time.time() - t_start\n",
+    "    results['losses'].append(avg_loss)\n",
+    "    results['epoch_times'].append(epoch_time)\n",
+    "    print(f\"  Epoch {epoch+1}/{EPOCHS}: loss={avg_loss:.4f} ({epoch_time:.2f}s)\")\n",
+    "\n",
+    "results['final_loss'] = results['losses'][-1]\n",
+    "results['avg_epoch_time'] = sum(results['epoch_times']) / len(results['epoch_times'])\n",
+    "\n",
+    "with open(f'/home/dev/results_{EXP_NAME}.json', 'w') as f:\n",
+    "    json.dump(results, f)\n",
+    "print(f\"Results saved to /home/dev/results_{EXP_NAME}.json\")\n",
+    "SCRIPT\n",
+    "\"\"\")\n",
+    "\n",
+    "# Verify\n",
+    "result = base.exec(\"ls -la /home/dev/train.py && python3 -c 'import wandb, timm; print(\\\"packages OK\\\")'\")\n",
+    "print(result.stdout.strip())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 2: Shut Down and Clone the Disk\n",
+    "\n",
+    "Cancel the base reservation (disk is snapshotted automatically),\n",
+    "then clone it for a parallel experiment."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "t0 = time.time()\n",
+    "base.cancel()\n",
+    "cancel_time = time.time() - t0\n",
+    "print(f\"Base cancelled in {cancel_time:.1f}s (disk snapshotted)\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Clone the disk for the second experiment\n",
+    "t0 = time.time()\n",
+    "client.clone_disk(\"experiment-base\", \"experiment-variant\")\n",
+    "clone_time = time.time() - t0\n",
+    "print(f\"Disk cloned in {clone_time:.1f}s\")\n",
+    "\n",
+    "# Show both disks\n",
+    "for disk in client.disks():\n",
+    "    if 'experiment' in disk.name:\n",
+    "        print(f\"  {disk.name:25s}  {disk.size_gb}GB  {disk.snapshot_count} snapshots\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 3: Run Parallel Experiments\n",
+    "\n",
+    "Launch two reservations simultaneously \u2014 one on the original disk (high LR),\n",
+    "one on the cloned disk (low LR). Both have the same training script pre-installed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "experiments = [\n",
+    "    {\"name\": \"high-lr\",  \"disk\": \"experiment-base\",    \"env\": \"LR=0.01 BATCH_SIZE=128 EPOCHS=5 EXP_NAME=high_lr\"},\n",
+    "    {\"name\": \"low-lr\",   \"disk\": \"experiment-variant\",  \"env\": \"LR=0.0001 BATCH_SIZE=32 EPOCHS=5 EXP_NAME=low_lr\"},\n",
+    "]\n",
+    "\n",
+    "def run_experiment(exp):\n",
+    "    \"\"\"Reserve GPU, run training, collect results, cancel.\"\"\"\n",
+    "    timings = {}\n",
+    "    \n",
+    "    # Reserve\n",
+    "    t0 = time.time()\n",
+    "    sb = client.reserve(\n",
+    "        gpu_type=\"t4\",\n",
+    "        gpu_count=1,\n",
+    "        hours=0.5,\n",
+    "        disk_name=exp[\"disk\"],\n",
+    "        name=exp[\"name\"],\n",
+    "    )\n",
+    "    timings['reserve'] = time.time() - t0\n",
+    "    \n",
+    "    # Run training\n",
+    "    t0 = time.time()\n",
+    "    result = sb.exec(f\"{exp['env']} python3 /home/dev/train.py\", timeout=120)\n",
+    "    timings['train'] = time.time() - t0\n",
+    "    train_output = result.stdout.strip()\n",
+    "    \n",
+    "    # Collect results\n",
+    "    exp_name = exp['env'].split('EXP_NAME=')[1].split()[0]\n",
+    "    result = sb.exec(f\"cat /home/dev/results_{exp_name}.json\")\n",
+    "    import json\n",
+    "    results = json.loads(result.stdout.strip())\n",
+    "    \n",
+    "    # Cancel\n",
+    "    t0 = time.time()\n",
+    "    sb.cancel()\n",
+    "    timings['cancel'] = time.time() - t0\n",
+    "    \n",
+    "    return {\n",
+    "        'experiment': exp['name'],\n",
+    "        'timings': timings,\n",
+    "        'results': results,\n",
+    "        'train_output': train_output,\n",
+    "    }\n",
+    "\n",
+    "# Run both experiments in parallel\n",
+    "t_total = time.time()\n",
+    "with ThreadPoolExecutor(max_workers=2) as pool:\n",
+    "    futures = {pool.submit(run_experiment, exp): exp['name'] for exp in experiments}\n",
+    "    outputs = {}\n",
+    "    for future in as_completed(futures):\n",
+    "        name = futures[future]\n",
+    "        outputs[name] = future.result()\n",
+    "        print(f\"\u2705 {name} completed\")\n",
+    "\n",
+    "total_time = time.time() - t_total\n",
+    "print(f\"\\nBoth experiments completed in {total_time:.1f}s (parallel)\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 4: Compare Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"=\" * 60)\n",
+    "print(f\"{'Metric':<25s} {'High LR':>15s} {'Low LR':>15s}\")\n",
+    "print(\"=\" * 60)\n",
+    "\n",
+    "high = outputs['high-lr']['results']\n",
+    "low = outputs['low-lr']['results']\n",
+    "\n",
+    "print(f\"{'Learning Rate':<25s} {high['lr']:>15.4f} {low['lr']:>15.4f}\")\n",
+    "print(f\"{'Batch Size':<25s} {high['batch_size']:>15d} {low['batch_size']:>15d}\")\n",
+    "print(f\"{'Final Loss':<25s} {high['final_loss']:>15.4f} {low['final_loss']:>15.4f}\")\n",
+    "print(f\"{'Avg Epoch Time (s)':<25s} {high['avg_epoch_time']:>15.2f} {low['avg_epoch_time']:>15.2f}\")\n",
+    "print()\n",
+    "\n",
+    "# Loss progression\n",
+    "print(\"Loss progression:\")\n",
+    "for i in range(len(high['losses'])):\n",
+    "    print(f\"  Epoch {i+1}: high_lr={high['losses'][i]:.4f}  low_lr={low['losses'][i]:.4f}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 5: Timing Breakdown"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"\\n\u23f1\ufe0f  Timing Breakdown\")\n",
+    "print(\"=\" * 60)\n",
+    "print(f\"{'Phase':<25s} {'High LR':>15s} {'Low LR':>15s}\")\n",
+    "print(\"-\" * 60)\n",
+    "\n",
+    "for phase in ['reserve', 'train', 'cancel']:\n",
+    "    h = outputs['high-lr']['timings'][phase]\n",
+    "    l = outputs['low-lr']['timings'][phase]\n",
+    "    print(f\"{phase.capitalize():<25s} {h:>14.1f}s {l:>14.1f}s\")\n",
+    "\n",
+    "print(\"-\" * 60)\n",
+    "h_total = sum(outputs['high-lr']['timings'].values())\n",
+    "l_total = sum(outputs['low-lr']['timings'].values())\n",
+    "print(f\"{'Total (sequential)':<25s} {h_total:>14.1f}s {l_total:>14.1f}s\")\n",
+    "print(f\"{'Total (parallel)':<25s} {total_time:>14.1f}s {'\u2014':>15s}\")\n",
+    "print(f\"{'Speedup':<25s} {(h_total + l_total) / total_time:>14.1f}x {'':>15s}\")\n",
+    "print()\n",
+    "print(f\"Disk clone time: {clone_time:.1f}s\")\n",
+    "print(f\"Base setup + cancel: {reserve_time + cancel_time:.1f}s\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Cleanup\n",
+    "\n",
+    "Remove the experiment disks if you don't need them."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Uncomment to delete experiment disks:\n",
+    "# client.delete_disk(\"experiment-base\")\n",
+    "# client.delete_disk(\"experiment-variant\")\n",
+    "print(\"Done! Disks preserved for inspection.\")\n",
+    "print(\"Delete with: client.delete_disk('experiment-base')\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/sdk/python/src/gpu_dev/_backend/aws.py

@@ -282,6 +282,39 @@ class AwsBackend:
             for item in resp.get("Items", [])
         ]
 
+    def clone_disk(self, user_id: str, source_disk: str, target_disk: str) -> str:
+        import uuid
+        from datetime import datetime, timezone
+        operation_id = str(uuid.uuid4())
+        self._sqs.send_message(
+            QueueUrl=self._get_queue_url(),
+            MessageBody=json.dumps({
+                "action": "clone_disk",
+                "operation_id": operation_id,
+                "user_id": user_id,
+                "source_disk": source_disk,
+                "target_disk": target_disk,
+                "requested_at": datetime.now(timezone.utc).isoformat(),
+            }),
+        )
+        return operation_id
+
+    def delete_disk(self, user_id: str, disk_name: str) -> str:
+        import uuid
+        from datetime import datetime, timezone
+        operation_id = str(uuid.uuid4())
+        self._sqs.send_message(
+            QueueUrl=self._get_queue_url(),
+            MessageBody=json.dumps({
+                "action": "delete_disk",
+                "operation_id": operation_id,
+                "user_id": user_id,
+                "disk_name": disk_name,
+                "requested_at": datetime.now(timezone.utc).isoformat(),
+            }),
+        )
+        return operation_id
+
     def add_user(self, reservation_id: str, user_id: str, github_username: str) -> bool:
         message = {
             "type": "add_user",

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/sdk/python/src/gpu_dev/_backend/protocol.py

@@ -48,6 +48,14 @@ class Backend(Protocol):
         """List persistent disks for a user."""
         ...
 
+    def clone_disk(self, user_id: str, source_disk: str, target_disk: str) -> str:
+        """Clone a disk. Returns operation_id."""
+        ...
+
+    def delete_disk(self, user_id: str, disk_name: str) -> str:
+        """Delete a disk. Returns operation_id."""
+        ...
+
     def add_user(self, reservation_id: str, user_id: str, github_username: str) -> bool:
         """Grant SSH access to another user."""
         ...

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/sdk/python/src/gpu_dev/_sync/client.py

@@ -244,6 +244,42 @@ class GpuDev:
         user_info = self._auth()
         return self._backend.list_disks(user_info["user_id"])
 
+    def clone_disk(self, source: str, target: str, *, poll: bool = True, timeout: int = 120) -> str:
+        """Clone a persistent disk.
+
+        Args:
+            source: Name of the source disk.
+            target: Name for the new cloned disk.
+            poll: Wait for the clone to complete (default True).
+            timeout: Max seconds to wait when polling.
+
+        Returns:
+            Operation ID.
+        """
+        user_info = self._auth()
+        op_id = self._backend.clone_disk(user_info["user_id"], source, target)
+        if poll:
+            import time
+            deadline = time.time() + timeout
+            while time.time() < deadline:
+                disks = self._backend.list_disks(user_info["user_id"])
+                if any(d.name == target for d in disks):
+                    return op_id
+                time.sleep(2)
+        return op_id
+
+    def delete_disk(self, name: str) -> str:
+        """Delete a persistent disk.
+
+        Args:
+            name: Disk name to delete.
+
+        Returns:
+            Operation ID.
+        """
+        user_info = self._auth()
+        return self._backend.delete_disk(user_info["user_id"], name)
+
     def search_logs(
         self,
         reservation_id: str,

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/terraform-gpu-devservers/availability.tf

@@ -25,7 +25,7 @@ resource "aws_lambda_function" "availability_updater" {
   function_name    = "${var.prefix}-availability-updater"
   role             = aws_iam_role.availability_updater_role.arn
   handler          = "index.handler"
-  runtime          = "python3.11"
+  runtime          = "python3.13"
   timeout                        = 300
   # 1769 MB is the sweet spot — Lambda allocates one full vCPU at this threshold.
   # Beyond 1769 MB you get fractional second vCPUs (less linear gain), and our work is single-threaded.

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/terraform-gpu-devservers/expiry.tf

@@ -15,6 +15,7 @@ resource "aws_lambda_function" "reservation_expiry" {
   environment {
     variables = {
       RESERVATIONS_TABLE                 = aws_dynamodb_table.gpu_reservations.name
+      DISKS_TABLE_NAME                   = aws_dynamodb_table.disks.name
       EKS_CLUSTER_NAME                   = aws_eks_cluster.gpu_dev_cluster.name
       REGION                             = local.current_config.aws_region
       WARNING_MINUTES                    = "30"  # Warn 30 minutes before expiry

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/terraform-gpu-devservers/lambda/availability_updater/index.py

@@ -330,6 +330,7 @@ def update_gpu_availability(gpu_type: str, k8s_client=None, active_reservations=
 
                 single_node_max = 0  # Max available on any single node
                 schedulable_total_gpus = 0  # Total GPUs on schedulable (non-cordoned) nodes
+                full_node_gpu_counts = []  # Track actual GPU count per full node (accounts for MIG)
                 for node in nodes.items:
                     if is_node_ready_and_schedulable(node):
                         available_on_node = get_available_gpus_on_node(v1, node, gpu_type)
@@ -349,24 +350,24 @@ def update_gpu_availability(gpu_type: str, k8s_client=None, active_reservations=
                         # Count as full node if all GPUs are available
                         if total_on_node > 0 and available_on_node == total_on_node:
                             full_nodes_available += 1
+                            full_node_gpu_counts.append(total_on_node)
 
                 total_gpus = schedulable_total_gpus
                 # For MIG SKUs override running_instances to the number of MIG-partitioned nodes
                 if is_mig_sku:
                     running_instances = sum(1 for n in nodes.items if is_node_ready_and_schedulable(n) and int((n.status.allocatable or {}).get(resource_name, "0")) > 0)
 
-                # Calculate max reservable considering multinode scenarios
-                # Only high-end GPU types support multinode (up to 4 nodes = 32 GPUs)
+                # Calculate max reservable using actual per-node GPU counts (not ASG gpus_per_instance)
+                # This correctly accounts for MIG-configured nodes that have fewer full GPUs
                 multinode_gpu_types = ['h100', 'h200', 'b200', 'a100']
-                if gpu_type in multinode_gpu_types and gpus_per_instance == 8:
-                    max_nodes = min(4, full_nodes_available)  # Up to 4 nodes
-                    max_reservable = max_nodes * gpus_per_instance  # e.g., 4 * 8 = 32 GPUs
+                if gpu_type in multinode_gpu_types and full_node_gpu_counts:
+                    # Sum the top N full nodes (up to 4 for multinode)
+                    sorted_counts = sorted(full_node_gpu_counts, reverse=True)
+                    max_reservable = sum(sorted_counts[:4])
 
-                    # If no full nodes available, fall back to single node max
                     if max_reservable == 0:
                         max_reservable = single_node_max
                 else:
-                    # For all other GPU types (T4, L4, T4-small, etc.), only single node
                     max_reservable = single_node_max
 
                 logger.info(f"Found {full_nodes_available} full nodes available for {gpu_type}, max reservable: {max_reservable} (single node max: {single_node_max})")

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/terraform-gpu-devservers/lambda/reservation_expiry/index.py

@@ -892,6 +892,7 @@ def handler(event, context):
             logger.error(f"Error cleaning up soft-deleted snapshots: {e}")
             deleted_snapshot_count = 0
 
+
         return {
             "statusCode": 200,
             "body": json.dumps(

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/terraform-gpu-devservers/lambda/reservation_processor/index.py

@@ -2895,28 +2895,42 @@ def allocate_gpu_resources(reservation_id: str, request: dict[str, Any], trace_d
             return keys
 
         def _check_nvme_cache_on_node(_target_node, _user_id):
-            """Check if target node has NVMe cache for this user via DaemonSet pod."""
-            if not _target_node:
-                return False
+            """Check if target node (or any node of this GPU type) has NVMe cache."""
             try:
                 v1 = client.CoreV1Api(k8s_client)
                 cache_dir = f"/mnt/nvme/user-cache/{_nvme_cache_user_dir(_user_id)}"
+
+                if _target_node:
+                    field_sel = f"spec.nodeName={_target_node},status.phase=Running"
+                else:
+                    field_sel = "status.phase=Running"
+
                 pods = v1.list_namespaced_pod(
                     "kube-system",
-                    field_selector=f"spec.nodeName={_target_node},status.phase=Running",
+                    field_selector=field_sel,
                     label_selector="app=image-prepuller",
                 ).items
                 if not pods:
+                    logger.info(f"NVMe cache check: no prepuller pods found")
                     return False
-                result = stream.stream(
-                    v1.connect_get_namespaced_pod_exec,
-                    pods[0].metadata.name, "kube-system",
-                    container="pause",
-                    command=["test", "-d", cache_dir],
-                    stderr=True, stdout=True, stdin=False, tty=False,
-                )
-                return True
-            except Exception:
+
+                for pod in pods[:3]:
+                    try:
+                        stream.stream(
+                            v1.connect_get_namespaced_pod_exec,
+                            pod.metadata.name, "kube-system",
+                            container="pause",
+                            command=["test", "-d", cache_dir],
+                            stderr=True, stdout=True, stdin=False, tty=False,
+                        )
+                        logger.info(f"NVMe cache HIT on {pod.spec.node_name}")
+                        return True
+                    except Exception:
+                        continue
+                logger.info(f"NVMe cache MISS for {_user_id}")
+                return False
+            except Exception as e:
+                logger.warning(f"NVMe cache check error: {e}")
                 return False
 
         def _setup_disk():

{gpu_dev-0.6.5 → gpu_dev-0.6.6}/terraform-gpu-devservers/lambda/shared/dns_utils.py

@@ -144,8 +144,7 @@ def get_existing_dns_names() -> List[str]:
         return []
 
     try:
-        dynamodb = boto3.resource("dynamodb")
-        table = dynamodb.Table(table_name)
+        table = _get_domain_mappings_table(table_name)
 
         # Scan for all domain mappings
         response = table.scan()

gpu_dev-0.6.6/terraform-gpu-devservers/lambda/shared/requirements.txt

@@ -0,0 +1,3 @@
+kubernetes==28.1.0
+boto3==1.34.0
+urllib3<2.0

gpu_dev-0.6.5/terraform-gpu-devservers/lambda/availability_updater/requirements.txt

@@ -1,2 +0,0 @@
-boto3>=1.26.0
-kubernetes>=24.2.0