gpu-dev 0.6.4__tar.gz → 0.6.6__tar.gz

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.6.4
+Version: 0.6.6
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpu-dev
-Version: 0.6.4
+Version: 0.6.6
 Summary: CLI tool for PyTorch GPU developer server reservations
 Author: PyTorch Team
 Requires-Python: >=3.10

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/cli-tools/gpu-dev-cli/gpu_dev.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ sdk/python/README.md
 sdk/python/pyproject.toml
 sdk/python/examples/batch_multi_gpu.py
 sdk/python/examples/interactive_debug.py
+sdk/python/examples/parallel_experiments.ipynb
 sdk/python/examples/quickstart.ipynb
 sdk/python/examples/run_tests.py
 sdk/python/examples/submit_job.py

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/cli-tools/gpu-dev-cli/gpu_dev_cli/disks.py

@@ -154,12 +154,8 @@ def list_disks(user_id: str, config: Config) -> List[Dict]:
     List all disks for a user.
     Returns list of disk info dicts with: name, size, last_used, created_at, snapshot_count, in_use, reservation_id
     """
-    ec2_client = get_ec2_client(config)
-    dynamodb = get_dynamodb_resource(config)
-
-    # Query DynamoDB disks table for this user's disks (with pagination)
-    disks_table_name = config.disks_table if hasattr(config, 'disks_table') else f"{config.queue_name.rsplit('-', 1)[0]}-disks"
-    disks_table = dynamodb.Table(disks_table_name)
+    dynamodb = config.dynamodb
+    disks_table = dynamodb.Table(config.disks_table)
 
     dynamodb_disks = []
     response = disks_table.query(
@@ -208,9 +204,6 @@ def list_disks(user_id: str, config: Config) -> List[Dict]:
         is_deleted = disk_item.get('is_deleted', False)
         delete_date = disk_item.get('delete_date')
 
-        # Check current in_use status (check dynamically from reservations table)
-        is_in_use, reservation_id = get_disk_in_use_status(disk_name, user_id, config)
-
         disks.append({
             'name': disk_name,
             'size_gb': size_gb,
@@ -219,13 +212,44 @@ def list_disks(user_id: str, config: Config) -> List[Dict]:
             'last_used': last_used,
             'snapshot_count': snapshot_count,
             'pending_snapshot_count': pending_snapshot_count,
-            'in_use': is_in_use,
+            'in_use': bool(disk_item.get('in_use', False)),
             'is_backing_up': is_backing_up,
-            'reservation_id': reservation_id,
+            'reservation_id': str(disk_item.get('attached_to_reservation', '')) or None,
             'is_deleted': is_deleted,
             'delete_date': delete_date,
         })
 
+    # Batch check: find all active reservations with disk_name set (single query)
+    try:
+        reservations_table = dynamodb.Table(config.reservations_table)
+        active_disks = {}
+        for status in ["active", "preparing", "queued", "pending"]:
+            resp = reservations_table.query(
+                IndexName="UserStatusIndex",
+                KeyConditionExpression="user_id = :uid AND #s = :status",
+                ExpressionAttributeNames={"#s": "status"},
+                ExpressionAttributeValues={":uid": user_id, ":status": status},
+                ProjectionExpression="reservation_id, disk_name",
+            )
+            for item in resp.get("Items", []):
+                dn = item.get("disk_name")
+                if dn:
+                    active_disks[dn] = str(item.get("reservation_id", ""))[:8]
+
+        for disk in disks:
+            if disk["name"] in active_disks:
+                disk["in_use"] = True
+                disk["reservation_id"] = active_disks[disk["name"]]
+    except Exception:
+        pass
+
+    # Filter out expired deleted disks (delete_date has passed)
+    today = datetime.now(timezone.utc).strftime('%Y-%m-%d')
+    disks = [
+        d for d in disks
+        if not (d.get('is_deleted') and d.get('delete_date') and str(d['delete_date']) <= today)
+    ]
+
     # Sort by last_used (most recent first)
     disks.sort(key=lambda d: d['last_used'] or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
 

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gpu-dev"
-version = "0.6.4"
+version = "0.6.6"
 description = "CLI tool for PyTorch GPU developer server reservations"
 authors = [{name = "PyTorch Team"}]
 readme = "cli-tools/gpu-dev-cli/README.md"

gpu_dev-0.6.6/sdk/python/examples/parallel_experiments.ipynb

@@ -0,0 +1,360 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Parallel Experiments with Persistent Disks\n",
+    "\n",
+    "This notebook demonstrates:\n",
+    "1. Creating a base environment on a persistent disk\n",
+    "2. Making changes (installing packages, modifying code)\n",
+    "3. Cloning the disk for parallel experiments\n",
+    "4. Running two experiments simultaneously on different GPUs\n",
+    "5. Comparing results and measuring timings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install -e .. -q"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
+    "from gpu_dev import GpuDev\n",
+    "\n",
+    "client = GpuDev()\n",
+    "print(f\"SDK v{__import__('gpu_dev').__version__}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 1: Create Base Environment on Persistent Disk\n",
+    "\n",
+    "Reserve a GPU with a persistent disk and set up the base experiment."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "t0 = time.time()\n",
+    "\n",
+    "base = client.reserve(\n",
+    "    gpu_type=\"t4\",\n",
+    "    gpu_count=1,\n",
+    "    hours=1,\n",
+    "    disk_name=\"experiment-base\",\n",
+    "    name=\"base-setup\",\n",
+    ")\n",
+    "\n",
+    "reserve_time = time.time() - t0\n",
+    "print(f\"Reserved in {reserve_time:.1f}s\")\n",
+    "print(f\"Disk: {base.disk_name}\")\n",
+    "print(f\"GPU:  {base.gpu_type} x{base.gpu_count}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Set up the base experiment: install packages + write training script\n",
+    "base.exec(\"pip install -q wandb timm\")\n",
+    "\n",
+    "# Write a parameterized training script\n",
+    "base.exec(r\"\"\"\n",
+    "cat > /home/dev/train.py << 'SCRIPT'\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import time\n",
+    "import json\n",
+    "import os\n",
+    "import sys\n",
+    "\n",
+    "# Read experiment config from env\n",
+    "LR = float(os.environ.get('LR', '0.001'))\n",
+    "BATCH_SIZE = int(os.environ.get('BATCH_SIZE', '64'))\n",
+    "EPOCHS = int(os.environ.get('EPOCHS', '5'))\n",
+    "EXP_NAME = os.environ.get('EXP_NAME', 'default')\n",
+    "\n",
+    "print(f\"Experiment: {EXP_NAME}\")\n",
+    "print(f\"Config: lr={LR}, batch_size={BATCH_SIZE}, epochs={EPOCHS}\")\n",
+    "print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n",
+    "print(f\"PyTorch: {torch.__version__}\")\n",
+    "\n",
+    "# Simple CNN on synthetic data\n",
+    "model = nn.Sequential(\n",
+    "    nn.Conv2d(3, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),\n",
+    "    nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),\n",
+    "    nn.Flatten(), nn.Linear(64 * 8 * 8, 10)\n",
+    ").cuda()\n",
+    "\n",
+    "optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n",
+    "criterion = nn.CrossEntropyLoss()\n",
+    "\n",
+    "results = {'experiment': EXP_NAME, 'lr': LR, 'batch_size': BATCH_SIZE, 'losses': [], 'epoch_times': []}\n",
+    "\n",
+    "for epoch in range(EPOCHS):\n",
+    "    t_start = time.time()\n",
+    "    epoch_loss = 0\n",
+    "    for step in range(50):\n",
+    "        x = torch.randn(BATCH_SIZE, 3, 32, 32, device='cuda')\n",
+    "        y = torch.randint(0, 10, (BATCH_SIZE,), device='cuda')\n",
+    "        loss = criterion(model(x), y)\n",
+    "        optimizer.zero_grad()\n",
+    "        loss.backward()\n",
+    "        optimizer.step()\n",
+    "        epoch_loss += loss.item()\n",
+    "    avg_loss = epoch_loss / 50\n",
+    "    epoch_time = time.time() - t_start\n",
+    "    results['losses'].append(avg_loss)\n",
+    "    results['epoch_times'].append(epoch_time)\n",
+    "    print(f\"  Epoch {epoch+1}/{EPOCHS}: loss={avg_loss:.4f} ({epoch_time:.2f}s)\")\n",
+    "\n",
+    "results['final_loss'] = results['losses'][-1]\n",
+    "results['avg_epoch_time'] = sum(results['epoch_times']) / len(results['epoch_times'])\n",
+    "\n",
+    "with open(f'/home/dev/results_{EXP_NAME}.json', 'w') as f:\n",
+    "    json.dump(results, f)\n",
+    "print(f\"Results saved to /home/dev/results_{EXP_NAME}.json\")\n",
+    "SCRIPT\n",
+    "\"\"\")\n",
+    "\n",
+    "# Verify\n",
+    "result = base.exec(\"ls -la /home/dev/train.py && python3 -c 'import wandb, timm; print(\\\"packages OK\\\")'\")\n",
+    "print(result.stdout.strip())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 2: Shut Down and Clone the Disk\n",
+    "\n",
+    "Cancel the base reservation (disk is snapshotted automatically),\n",
+    "then clone it for a parallel experiment."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "t0 = time.time()\n",
+    "base.cancel()\n",
+    "cancel_time = time.time() - t0\n",
+    "print(f\"Base cancelled in {cancel_time:.1f}s (disk snapshotted)\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Clone the disk for the second experiment\n",
+    "t0 = time.time()\n",
+    "client.clone_disk(\"experiment-base\", \"experiment-variant\")\n",
+    "clone_time = time.time() - t0\n",
+    "print(f\"Disk cloned in {clone_time:.1f}s\")\n",
+    "\n",
+    "# Show both disks\n",
+    "for disk in client.disks():\n",
+    "    if 'experiment' in disk.name:\n",
+    "        print(f\"  {disk.name:25s}  {disk.size_gb}GB  {disk.snapshot_count} snapshots\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 3: Run Parallel Experiments\n",
+    "\n",
+    "Launch two reservations simultaneously \u2014 one on the original disk (high LR),\n",
+    "one on the cloned disk (low LR). Both have the same training script pre-installed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "experiments = [\n",
+    "    {\"name\": \"high-lr\",  \"disk\": \"experiment-base\",    \"env\": \"LR=0.01 BATCH_SIZE=128 EPOCHS=5 EXP_NAME=high_lr\"},\n",
+    "    {\"name\": \"low-lr\",   \"disk\": \"experiment-variant\",  \"env\": \"LR=0.0001 BATCH_SIZE=32 EPOCHS=5 EXP_NAME=low_lr\"},\n",
+    "]\n",
+    "\n",
+    "def run_experiment(exp):\n",
+    "    \"\"\"Reserve GPU, run training, collect results, cancel.\"\"\"\n",
+    "    timings = {}\n",
+    "    \n",
+    "    # Reserve\n",
+    "    t0 = time.time()\n",
+    "    sb = client.reserve(\n",
+    "        gpu_type=\"t4\",\n",
+    "        gpu_count=1,\n",
+    "        hours=0.5,\n",
+    "        disk_name=exp[\"disk\"],\n",
+    "        name=exp[\"name\"],\n",
+    "    )\n",
+    "    timings['reserve'] = time.time() - t0\n",
+    "    \n",
+    "    # Run training\n",
+    "    t0 = time.time()\n",
+    "    result = sb.exec(f\"{exp['env']} python3 /home/dev/train.py\", timeout=120)\n",
+    "    timings['train'] = time.time() - t0\n",
+    "    train_output = result.stdout.strip()\n",
+    "    \n",
+    "    # Collect results\n",
+    "    exp_name = exp['env'].split('EXP_NAME=')[1].split()[0]\n",
+    "    result = sb.exec(f\"cat /home/dev/results_{exp_name}.json\")\n",
+    "    import json\n",
+    "    results = json.loads(result.stdout.strip())\n",
+    "    \n",
+    "    # Cancel\n",
+    "    t0 = time.time()\n",
+    "    sb.cancel()\n",
+    "    timings['cancel'] = time.time() - t0\n",
+    "    \n",
+    "    return {\n",
+    "        'experiment': exp['name'],\n",
+    "        'timings': timings,\n",
+    "        'results': results,\n",
+    "        'train_output': train_output,\n",
+    "    }\n",
+    "\n",
+    "# Run both experiments in parallel\n",
+    "t_total = time.time()\n",
+    "with ThreadPoolExecutor(max_workers=2) as pool:\n",
+    "    futures = {pool.submit(run_experiment, exp): exp['name'] for exp in experiments}\n",
+    "    outputs = {}\n",
+    "    for future in as_completed(futures):\n",
+    "        name = futures[future]\n",
+    "        outputs[name] = future.result()\n",
+    "        print(f\"\u2705 {name} completed\")\n",
+    "\n",
+    "total_time = time.time() - t_total\n",
+    "print(f\"\\nBoth experiments completed in {total_time:.1f}s (parallel)\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 4: Compare Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"=\" * 60)\n",
+    "print(f\"{'Metric':<25s} {'High LR':>15s} {'Low LR':>15s}\")\n",
+    "print(\"=\" * 60)\n",
+    "\n",
+    "high = outputs['high-lr']['results']\n",
+    "low = outputs['low-lr']['results']\n",
+    "\n",
+    "print(f\"{'Learning Rate':<25s} {high['lr']:>15.4f} {low['lr']:>15.4f}\")\n",
+    "print(f\"{'Batch Size':<25s} {high['batch_size']:>15d} {low['batch_size']:>15d}\")\n",
+    "print(f\"{'Final Loss':<25s} {high['final_loss']:>15.4f} {low['final_loss']:>15.4f}\")\n",
+    "print(f\"{'Avg Epoch Time (s)':<25s} {high['avg_epoch_time']:>15.2f} {low['avg_epoch_time']:>15.2f}\")\n",
+    "print()\n",
+    "\n",
+    "# Loss progression\n",
+    "print(\"Loss progression:\")\n",
+    "for i in range(len(high['losses'])):\n",
+    "    print(f\"  Epoch {i+1}: high_lr={high['losses'][i]:.4f}  low_lr={low['losses'][i]:.4f}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Step 5: Timing Breakdown"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"\\n\u23f1\ufe0f  Timing Breakdown\")\n",
+    "print(\"=\" * 60)\n",
+    "print(f\"{'Phase':<25s} {'High LR':>15s} {'Low LR':>15s}\")\n",
+    "print(\"-\" * 60)\n",
+    "\n",
+    "for phase in ['reserve', 'train', 'cancel']:\n",
+    "    h = outputs['high-lr']['timings'][phase]\n",
+    "    l = outputs['low-lr']['timings'][phase]\n",
+    "    print(f\"{phase.capitalize():<25s} {h:>14.1f}s {l:>14.1f}s\")\n",
+    "\n",
+    "print(\"-\" * 60)\n",
+    "h_total = sum(outputs['high-lr']['timings'].values())\n",
+    "l_total = sum(outputs['low-lr']['timings'].values())\n",
+    "print(f\"{'Total (sequential)':<25s} {h_total:>14.1f}s {l_total:>14.1f}s\")\n",
+    "print(f\"{'Total (parallel)':<25s} {total_time:>14.1f}s {'\u2014':>15s}\")\n",
+    "print(f\"{'Speedup':<25s} {(h_total + l_total) / total_time:>14.1f}x {'':>15s}\")\n",
+    "print()\n",
+    "print(f\"Disk clone time: {clone_time:.1f}s\")\n",
+    "print(f\"Base setup + cancel: {reserve_time + cancel_time:.1f}s\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Cleanup\n",
+    "\n",
+    "Remove the experiment disks if you don't need them."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Uncomment to delete experiment disks:\n",
+    "# client.delete_disk(\"experiment-base\")\n",
+    "# client.delete_disk(\"experiment-variant\")\n",
+    "print(\"Done! Disks preserved for inspection.\")\n",
+    "print(\"Delete with: client.delete_disk('experiment-base')\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/sdk/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gpu-dev-sdk"
-version = "0.1.0"
+version = "0.6.5"
 description = "Python SDK for GPU development server reservations"
 readme = "README.md"
 requires-python = ">=3.10"

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/sdk/python/src/gpu_dev/__init__.py

@@ -57,4 +57,4 @@ __all__ = [
     "ExecResult",
 ]
 
-__version__ = "0.1.0"
+__version__ = "0.6.5"

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/sdk/python/src/gpu_dev/_backend/aws.py

@@ -166,7 +166,7 @@ class AwsBackend:
             "no_persistent_disk": params.get("no_persistent_disk", False),
             "github_user": params.get("github_user", ""),
             "preserve_entrypoint": params.get("preserve_entrypoint", False),
-            "version": "0.6.0",
+            "version": __import__("gpu_dev").__version__,
         }
         if params.get("disk_name"):
             message["disk_name"] = params["disk_name"]
@@ -282,6 +282,39 @@ class AwsBackend:
             for item in resp.get("Items", [])
         ]
 
+    def clone_disk(self, user_id: str, source_disk: str, target_disk: str) -> str:
+        import uuid
+        from datetime import datetime, timezone
+        operation_id = str(uuid.uuid4())
+        self._sqs.send_message(
+            QueueUrl=self._get_queue_url(),
+            MessageBody=json.dumps({
+                "action": "clone_disk",
+                "operation_id": operation_id,
+                "user_id": user_id,
+                "source_disk": source_disk,
+                "target_disk": target_disk,
+                "requested_at": datetime.now(timezone.utc).isoformat(),
+            }),
+        )
+        return operation_id
+
+    def delete_disk(self, user_id: str, disk_name: str) -> str:
+        import uuid
+        from datetime import datetime, timezone
+        operation_id = str(uuid.uuid4())
+        self._sqs.send_message(
+            QueueUrl=self._get_queue_url(),
+            MessageBody=json.dumps({
+                "action": "delete_disk",
+                "operation_id": operation_id,
+                "user_id": user_id,
+                "disk_name": disk_name,
+                "requested_at": datetime.now(timezone.utc).isoformat(),
+            }),
+        )
+        return operation_id
+
     def add_user(self, reservation_id: str, user_id: str, github_username: str) -> bool:
         message = {
             "type": "add_user",

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/sdk/python/src/gpu_dev/_backend/protocol.py

@@ -48,6 +48,14 @@ class Backend(Protocol):
         """List persistent disks for a user."""
         ...
 
+    def clone_disk(self, user_id: str, source_disk: str, target_disk: str) -> str:
+        """Clone a disk. Returns operation_id."""
+        ...
+
+    def delete_disk(self, user_id: str, disk_name: str) -> str:
+        """Delete a disk. Returns operation_id."""
+        ...
+
     def add_user(self, reservation_id: str, user_id: str, github_username: str) -> bool:
         """Grant SSH access to another user."""
         ...

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/sdk/python/src/gpu_dev/_sync/client.py

@@ -244,6 +244,42 @@ class GpuDev:
         user_info = self._auth()
         return self._backend.list_disks(user_info["user_id"])
 
+    def clone_disk(self, source: str, target: str, *, poll: bool = True, timeout: int = 120) -> str:
+        """Clone a persistent disk.
+
+        Args:
+            source: Name of the source disk.
+            target: Name for the new cloned disk.
+            poll: Wait for the clone to complete (default True).
+            timeout: Max seconds to wait when polling.
+
+        Returns:
+            Operation ID.
+        """
+        user_info = self._auth()
+        op_id = self._backend.clone_disk(user_info["user_id"], source, target)
+        if poll:
+            import time
+            deadline = time.time() + timeout
+            while time.time() < deadline:
+                disks = self._backend.list_disks(user_info["user_id"])
+                if any(d.name == target for d in disks):
+                    return op_id
+                time.sleep(2)
+        return op_id
+
+    def delete_disk(self, name: str) -> str:
+        """Delete a persistent disk.
+
+        Args:
+            name: Disk name to delete.
+
+        Returns:
+            Operation ID.
+        """
+        user_info = self._auth()
+        return self._backend.delete_disk(user_info["user_id"], name)
+
     def search_logs(
         self,
         reservation_id: str,

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/terraform-gpu-devservers/availability.tf

@@ -25,7 +25,7 @@ resource "aws_lambda_function" "availability_updater" {
   function_name    = "${var.prefix}-availability-updater"
   role             = aws_iam_role.availability_updater_role.arn
   handler          = "index.handler"
-  runtime          = "python3.11"
+  runtime          = "python3.13"
   timeout                        = 300
   # 1769 MB is the sweet spot — Lambda allocates one full vCPU at this threshold.
   # Beyond 1769 MB you get fractional second vCPUs (less linear gain), and our work is single-threaded.

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/terraform-gpu-devservers/expiry.tf

@@ -15,6 +15,7 @@ resource "aws_lambda_function" "reservation_expiry" {
   environment {
     variables = {
       RESERVATIONS_TABLE                 = aws_dynamodb_table.gpu_reservations.name
+      DISKS_TABLE_NAME                   = aws_dynamodb_table.disks.name
       EKS_CLUSTER_NAME                   = aws_eks_cluster.gpu_dev_cluster.name
       REGION                             = local.current_config.aws_region
       WARNING_MINUTES                    = "30"  # Warn 30 minutes before expiry

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/terraform-gpu-devservers/lambda/availability_updater/index.py

@@ -330,6 +330,7 @@ def update_gpu_availability(gpu_type: str, k8s_client=None, active_reservations=
 
                 single_node_max = 0  # Max available on any single node
                 schedulable_total_gpus = 0  # Total GPUs on schedulable (non-cordoned) nodes
+                full_node_gpu_counts = []  # Track actual GPU count per full node (accounts for MIG)
                 for node in nodes.items:
                     if is_node_ready_and_schedulable(node):
                         available_on_node = get_available_gpus_on_node(v1, node, gpu_type)
@@ -349,24 +350,24 @@ def update_gpu_availability(gpu_type: str, k8s_client=None, active_reservations=
                         # Count as full node if all GPUs are available
                         if total_on_node > 0 and available_on_node == total_on_node:
                             full_nodes_available += 1
+                            full_node_gpu_counts.append(total_on_node)
 
                 total_gpus = schedulable_total_gpus
                 # For MIG SKUs override running_instances to the number of MIG-partitioned nodes
                 if is_mig_sku:
                     running_instances = sum(1 for n in nodes.items if is_node_ready_and_schedulable(n) and int((n.status.allocatable or {}).get(resource_name, "0")) > 0)
 
-                # Calculate max reservable considering multinode scenarios
-                # Only high-end GPU types support multinode (up to 4 nodes = 32 GPUs)
+                # Calculate max reservable using actual per-node GPU counts (not ASG gpus_per_instance)
+                # This correctly accounts for MIG-configured nodes that have fewer full GPUs
                 multinode_gpu_types = ['h100', 'h200', 'b200', 'a100']
-                if gpu_type in multinode_gpu_types and gpus_per_instance == 8:
-                    max_nodes = min(4, full_nodes_available)  # Up to 4 nodes
-                    max_reservable = max_nodes * gpus_per_instance  # e.g., 4 * 8 = 32 GPUs
+                if gpu_type in multinode_gpu_types and full_node_gpu_counts:
+                    # Sum the top N full nodes (up to 4 for multinode)
+                    sorted_counts = sorted(full_node_gpu_counts, reverse=True)
+                    max_reservable = sum(sorted_counts[:4])
 
-                    # If no full nodes available, fall back to single node max
                     if max_reservable == 0:
                         max_reservable = single_node_max
                 else:
-                    # For all other GPU types (T4, L4, T4-small, etc.), only single node
                     max_reservable = single_node_max
 
                 logger.info(f"Found {full_nodes_available} full nodes available for {gpu_type}, max reservable: {max_reservable} (single node max: {single_node_max})")

{gpu_dev-0.6.4 → gpu_dev-0.6.6}/terraform-gpu-devservers/lambda/reservation_expiry/index.py

@@ -892,6 +892,7 @@ def handler(event, context):
             logger.error(f"Error cleaning up soft-deleted snapshots: {e}")
             deleted_snapshot_count = 0
 
+
         return {
             "statusCode": 200,
             "body": json.dumps(