dh-cli 0.12.3__tar.gz → 0.12.4__tar.gz

{dh_cli-0.12.3 → dh_cli-0.12.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dh-cli
-Version: 0.12.3
+Version: 0.12.4
 Summary: Dayhoff Labs developer CLI
 Author-email: Dayhoff Labs <dev@dayhofflabs.com>
 License: # PolyForm Noncommercial License 1.0.0

{dh_cli-0.12.3 → dh_cli-0.12.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "dh-cli"
-version = "0.12.3"
+version = "0.12.4"
 description = "Dayhoff Labs developer CLI"
 requires-python = ">=3.11"
 readme = "README.md"

{dh_cli-0.12.3 → dh_cli-0.12.4}/src/dh_cli/engines_studios/engine_commands.py

@@ -8,8 +8,65 @@ import click
 from .api_client import StudioManagerClient
 from .auth import check_aws_auth, detect_aws_environment, get_aws_username
 from .progress import format_idle_state, format_time_ago, wait_with_progress
+from .retry import retry_on_capacity
 from .ssh_config import update_ssh_config_silent
 
+# Default capacity-retry budget: transient InsufficientInstanceCapacity in our
+# G-family AZs usually clears within a few minutes (see topic-quota-and-caps.md).
+_DEFAULT_RETRY_TIMEOUT = 600  # seconds (~10 min)
+_DEFAULT_RETRY_INTERVAL = 30  # seconds
+
+
+def _retry_options(func):
+    """Shared --retry / --no-retry / interval options for launch and start."""
+    func = click.option(
+        "--no-retry",
+        is_flag=True,
+        help="Fail immediately on capacity errors instead of retrying.",
+    )(func)
+    func = click.option(
+        "--retry-timeout",
+        type=int,
+        default=_DEFAULT_RETRY_TIMEOUT,
+        show_default=True,
+        help="Max seconds to keep retrying transient capacity errors (0 disables).",
+    )(func)
+    func = click.option(
+        "--retry-interval",
+        type=int,
+        default=_DEFAULT_RETRY_INTERVAL,
+        show_default=True,
+        help="Seconds to wait between capacity-retry attempts.",
+    )(func)
+    return func
+
+
+def _make_capacity_retry(engine_type: str, no_retry: bool, timeout: int, interval: int):
+    """Build a retry wrapper that echoes status between capacity retries.
+
+    Returns a callable taking a zero-arg `action` and running it with bounded
+    capacity-retry. With `no_retry` (or timeout 0) it runs the action once.
+    """
+    effective_timeout = 0 if no_retry else timeout
+
+    def on_retry(attempt: int, error: BaseException, wait_seconds: float) -> None:
+        click.echo(
+            f"⏳ Attempt {attempt} failed: no {engine_type} capacity available "
+            f"right now (AWS-side). Retrying in {int(wait_seconds)}s… "
+            f"(Ctrl+C to cancel)",
+            err=True,
+        )
+
+    def run(action):
+        return retry_on_capacity(
+            action,
+            timeout=effective_timeout,
+            interval=interval,
+            on_retry=on_retry,
+        )
+
+    return run
+
 
 @click.group()
 def engine_cli():
@@ -44,6 +101,7 @@ def engine_cli():
     default=None,
     help="Environment (dev, sand, prod) - auto-detected if not specified",
 )
+@_retry_options
 def launch_engine(
     name: str,
     engine_type: str,
@@ -53,6 +111,9 @@ def launch_engine(
     no_wait: bool,
     skip_ssh_config: bool,
     env: Optional[str],
+    retry_interval: int,
+    retry_timeout: int,
+    no_retry: bool,
 ):
     """Launch a new engine for the current user (or specified user with --user flag)."""
 
@@ -83,8 +144,13 @@ def launch_engine(
     click.echo(f"🚀 Launching {engine_type} engine '{name}' for {user}...")
 
     try:
-        # Launch the engine
-        engine = client.launch_engine(name=name, user=user, engine_type=engine_type, boot_disk_size=boot_disk_size)
+        # Launch the engine, retrying transient AWS capacity errors by default.
+        retry = _make_capacity_retry(engine_type, no_retry, retry_timeout, retry_interval)
+        engine = retry(
+            lambda: client.launch_engine(
+                name=name, user=user, engine_type=engine_type, boot_disk_size=boot_disk_size
+            )
+        )
 
         engine_id = engine["instance_id"]
         click.echo(f"✓ EC2 instance launched: {engine_id}")
@@ -128,6 +194,10 @@ def launch_engine(
                 if update_ssh_config_silent(client, env):
                     click.echo("✓ SSH config updated")
 
+    except KeyboardInterrupt:
+        click.echo("\n✗ Cancelled while waiting for capacity.", err=True)
+        raise click.Abort()
+
     except Exception as e:
         error_msg = str(e)
 
@@ -197,7 +267,17 @@ def launch_engine(
     default=None,
     help="Environment (dev, sand, prod) - auto-detected if not specified",
 )
-def start_engine(name_or_id: str, no_wait: bool, skip_ssh_config: bool, yes: bool, env: Optional[str]):
+@_retry_options
+def start_engine(
+    name_or_id: str,
+    no_wait: bool,
+    skip_ssh_config: bool,
+    yes: bool,
+    env: Optional[str],
+    retry_interval: int,
+    retry_timeout: int,
+    no_retry: bool,
+):
     """Start a stopped engine."""
 
     # Check AWS auth first to provide clear error messages
@@ -225,7 +305,13 @@ def start_engine(name_or_id: str, no_wait: bool, skip_ssh_config: bool, yes: boo
         engine_id = engine["instance_id"]
         engine_name = engine.get("name", engine_id)
 
-        result = client.start_engine(engine_id)
+        # Start the instance, retrying transient AWS capacity errors by default.
+        # Note: a stopped engine restarts in its ORIGINAL AZ, so retry here is a
+        # time-wait for that one AZ's capacity to return — it cannot fall to
+        # another AZ (unlike launch).
+        engine_type_label = engine.get("engine_type", "engine")
+        retry = _make_capacity_retry(engine_type_label, no_retry, retry_timeout, retry_interval)
+        result = retry(lambda: client.start_engine(engine_id))
 
         if "error" in result:
             click.echo(f"✗ Error: {result['error']}", err=True)
@@ -305,6 +391,10 @@ def start_engine(name_or_id: str, no_wait: bool, skip_ssh_config: bool, yes: boo
                 if update_ssh_config_silent(client, env):
                     click.echo("✓ SSH config updated")
 
+    except KeyboardInterrupt:
+        click.echo("\n✗ Cancelled while waiting for capacity.", err=True)
+        raise click.Abort()
+
     except Exception as e:
         click.echo(f"✗ Error: {e}", err=True)
         raise click.Abort()

dh_cli-0.12.4/src/dh_cli/engines_studios/retry.py

@@ -0,0 +1,79 @@
+"""Bounded capacity-retry for engine launch/start.
+
+AWS periodically has no on-demand G-family capacity in our AZs, surfacing as
+`InsufficientInstanceCapacity` (and, when a subnet is IP-starved,
+`InsufficientFreeAddressesInSubnet`). These are transient: capacity returns
+within minutes. This helper retries such failures within a wall-clock budget
+so users (and agents) don't have to re-run the command by hand.
+
+Quota errors (`VcpuLimitExceeded`), instance-count limits
+(`InstanceLimitExceeded`), and validation errors are NOT capacity problems and
+are never retried — they will not resolve by waiting.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Callable, Optional, TypeVar
+
+T = TypeVar("T")
+
+# Substrings that mark a transient, retryable AWS capacity/IP shortage.
+_CAPACITY_MARKERS = (
+    "InsufficientInstanceCapacity",
+    "InsufficientFreeAddressesInSubnet",
+)
+
+
+def is_capacity_error(error: BaseException) -> bool:
+    """True if `error` is a transient AWS capacity/IP shortage worth retrying."""
+    message = str(error)
+    return any(marker in message for marker in _CAPACITY_MARKERS)
+
+
+def retry_on_capacity(
+    action: Callable[[], T],
+    *,
+    timeout: float,
+    interval: float,
+    on_retry: Optional[Callable[[int, BaseException, float], None]] = None,
+    monotonic: Callable[[], float] = time.monotonic,
+    sleep: Callable[[float], None] = time.sleep,
+) -> T:
+    """Call `action`, retrying transient capacity errors within a budget.
+
+    Args:
+        action: Zero-arg callable performing the launch/start attempt.
+        timeout: Total wall-clock budget in seconds. A new attempt is only
+            started while elapsed time is strictly below this. ``0`` yields a
+            single attempt (no retries).
+        interval: Seconds to wait between attempts.
+        on_retry: Optional callback invoked before each sleep with
+            ``(attempt_number, error, wait_seconds)`` — used to print status.
+        monotonic: Monotonic clock source (injected for testing).
+        sleep: Sleep function (injected for testing).
+
+    Returns:
+        Whatever `action` returns on its first success.
+
+    Raises:
+        The last capacity error if the budget is exhausted, or immediately
+        re-raises any non-capacity error from `action`.
+    """
+    start = monotonic()
+    attempt = 0
+    last_error: Optional[BaseException] = None
+    while True:
+        attempt += 1
+        try:
+            return action()
+        except Exception as error:  # noqa: BLE001 — re-raised below unless retryable
+            if not is_capacity_error(error):
+                raise
+            last_error = error
+            # Stop if the next attempt would not begin within the budget.
+            if (monotonic() - start) + interval >= timeout:
+                raise last_error
+            if on_retry is not None:
+                on_retry(attempt, error, interval)
+            sleep(interval)

dh_cli-0.12.4/tests/engines_studios/test_engine_retry_commands.py

@@ -0,0 +1,133 @@
+"""Command-level tests for capacity-retry wiring in `dh engine launch/start`.
+
+These invoke the Click commands with a mocked `StudioManagerClient` and a
+patched sleep, asserting that a transient `InsufficientInstanceCapacity`
+is retried by default, that `--no-retry` disables it, and that non-capacity
+errors still fail fast.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+from click.testing import CliRunner
+
+from dh_cli.engines_studios import engine_commands as ec
+
+
+@pytest.fixture
+def runner():
+    return CliRunner()
+
+
+@pytest.fixture(autouse=True)
+def _fast_no_side_effects():
+    """Neutralize auth/env detection, SSH-config writes, and real sleeping."""
+    with (
+        patch.object(ec, "check_aws_auth", lambda: None),
+        patch.object(ec, "detect_aws_environment", lambda: "dev"),
+        patch.object(ec, "get_aws_username", lambda: "tester"),
+        patch.object(ec, "update_ssh_config_silent", lambda *a, **k: False),
+        # Retry helper sleeps via time.sleep in retry module; make it instant.
+        patch("dh_cli.engines_studios.retry.time.sleep", lambda s: None),
+    ):
+        yield
+
+
+def _capacity_error():
+    return RuntimeError(
+        "Failed to launch instance: An error occurred "
+        "(InsufficientInstanceCapacity) when calling the RunInstances operation"
+    )
+
+
+def test_launch_retries_capacity_then_succeeds(runner):
+    client = MagicMock()
+    # Fail twice with capacity, then succeed.
+    client.launch_engine.side_effect = [
+        _capacity_error(),
+        _capacity_error(),
+        {"instance_id": "i-abc", "instance_type": "g5.2xlarge", "state": "pending"},
+    ]
+
+    with patch.object(ec, "StudioManagerClient", return_value=client):
+        result = runner.invoke(
+            ec.launch_engine,
+            ["myeng", "--type", "a10g", "--no-wait", "--retry-interval", "1"],
+        )
+
+    assert result.exit_code == 0, result.output
+    assert client.launch_engine.call_count == 3
+    assert "i-abc" in result.output
+    assert "Retrying" in result.output
+
+
+def test_launch_no_retry_fails_fast_on_capacity(runner):
+    client = MagicMock()
+    client.launch_engine.side_effect = _capacity_error()
+
+    with patch.object(ec, "StudioManagerClient", return_value=client):
+        result = runner.invoke(
+            ec.launch_engine,
+            ["myeng", "--type", "a10g", "--no-wait", "--no-retry"],
+        )
+
+    assert result.exit_code != 0
+    assert client.launch_engine.call_count == 1
+    assert "insufficient EC2 capacity" in result.output
+
+
+def test_launch_does_not_retry_quota_error(runner):
+    client = MagicMock()
+    client.launch_engine.side_effect = RuntimeError("VcpuLimitExceeded: quota")
+
+    with patch.object(ec, "StudioManagerClient", return_value=client):
+        result = runner.invoke(
+            ec.launch_engine,
+            ["myeng", "--type", "a10g", "--no-wait", "--retry-interval", "1"],
+        )
+
+    assert result.exit_code != 0
+    assert client.launch_engine.call_count == 1
+    assert "quota exceeded" in result.output
+
+
+def test_start_retries_capacity_then_succeeds(runner):
+    client = MagicMock()
+    client.get_engine_by_name.return_value = {
+        "instance_id": "i-xyz",
+        "name": "myeng",
+    }
+    client.start_engine.side_effect = [
+        _capacity_error(),
+        {"instance_id": "i-xyz", "status": "pending"},
+    ]
+
+    with patch.object(ec, "StudioManagerClient", return_value=client):
+        result = runner.invoke(
+            ec.start_engine,
+            ["myeng", "--no-wait", "--retry-interval", "1"],
+        )
+
+    assert result.exit_code == 0, result.output
+    assert client.start_engine.call_count == 2
+    assert "Retrying" in result.output
+
+
+def test_start_no_retry_fails_fast_on_capacity(runner):
+    client = MagicMock()
+    client.get_engine_by_name.return_value = {
+        "instance_id": "i-xyz",
+        "name": "myeng",
+    }
+    client.start_engine.side_effect = _capacity_error()
+
+    with patch.object(ec, "StudioManagerClient", return_value=client):
+        result = runner.invoke(
+            ec.start_engine,
+            ["myeng", "--no-wait", "--no-retry"],
+        )
+
+    assert result.exit_code != 0
+    assert client.start_engine.call_count == 1

dh_cli-0.12.4/tests/engines_studios/test_retry.py

@@ -0,0 +1,182 @@
+"""Tests for the capacity-retry helper used by `dh engine launch` / `start`.
+
+The helper's whole job is: keep calling an action while it fails with an
+AWS *capacity* error, within a bounded wall-clock budget, and re-raise the
+last error once the budget is spent. Non-capacity errors must propagate
+immediately. Time is injected so the tests are instant and deterministic.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from dh_cli.engines_studios.retry import is_capacity_error, retry_on_capacity
+
+
+class FakeClock:
+    """Deterministic monotonic clock; `sleep` advances it, records intervals."""
+
+    def __init__(self) -> None:
+        self.now = 0.0
+        self.slept: list[float] = []
+
+    def monotonic(self) -> float:
+        return self.now
+
+    def sleep(self, seconds: float) -> None:
+        self.slept.append(seconds)
+        self.now += seconds
+
+
+def test_returns_immediately_on_success():
+    clock = FakeClock()
+    calls = []
+
+    def action():
+        calls.append(1)
+        return "ok"
+
+    result = retry_on_capacity(
+        action, timeout=600, interval=30, monotonic=clock.monotonic, sleep=clock.sleep
+    )
+
+    assert result == "ok"
+    assert len(calls) == 1
+    assert clock.slept == []
+
+
+def test_retries_capacity_error_then_succeeds():
+    clock = FakeClock()
+    attempts = {"n": 0}
+
+    def action():
+        attempts["n"] += 1
+        if attempts["n"] < 3:
+            raise RuntimeError(
+                "Failed to launch instance: An error occurred "
+                "(InsufficientInstanceCapacity) when calling RunInstances"
+            )
+        return "launched"
+
+    result = retry_on_capacity(
+        action, timeout=600, interval=30, monotonic=clock.monotonic, sleep=clock.sleep
+    )
+
+    assert result == "launched"
+    assert attempts["n"] == 3
+    # Slept between the two failures (not after success).
+    assert clock.slept == [30, 30]
+
+
+def test_non_capacity_error_propagates_without_retry():
+    clock = FakeClock()
+    attempts = {"n": 0}
+
+    def action():
+        attempts["n"] += 1
+        raise RuntimeError("VcpuLimitExceeded: vCPU quota exceeded")
+
+    with pytest.raises(RuntimeError, match="VcpuLimitExceeded"):
+        retry_on_capacity(
+            action, timeout=600, interval=30, monotonic=clock.monotonic, sleep=clock.sleep
+        )
+
+    assert attempts["n"] == 1
+    assert clock.slept == []
+
+
+def test_reraises_last_capacity_error_after_budget_exhausted():
+    clock = FakeClock()
+    attempts = {"n": 0}
+
+    def action():
+        attempts["n"] += 1
+        raise RuntimeError("InsufficientInstanceCapacity: no capacity")
+
+    with pytest.raises(RuntimeError, match="InsufficientInstanceCapacity"):
+        retry_on_capacity(
+            action, timeout=60, interval=30, monotonic=clock.monotonic, sleep=clock.sleep
+        )
+
+    # attempt@t=0 fails; next attempt (t=30) fits in the 60s budget -> sleep 30.
+    # attempt@t=30 fails; next attempt (t=60) would NOT be < 60 -> stop, no sleep.
+    assert attempts["n"] == 2
+    assert clock.slept == [30]
+
+
+def test_timeout_zero_means_single_attempt():
+    clock = FakeClock()
+    attempts = {"n": 0}
+
+    def action():
+        attempts["n"] += 1
+        raise RuntimeError("InsufficientInstanceCapacity")
+
+    with pytest.raises(RuntimeError, match="InsufficientInstanceCapacity"):
+        retry_on_capacity(
+            action, timeout=0, interval=30, monotonic=clock.monotonic, sleep=clock.sleep
+        )
+
+    assert attempts["n"] == 1
+    assert clock.slept == []
+
+
+def test_on_retry_callback_receives_attempt_and_error():
+    clock = FakeClock()
+    seen = []
+    attempts = {"n": 0}
+
+    def action():
+        attempts["n"] += 1
+        if attempts["n"] < 2:
+            raise RuntimeError("InsufficientInstanceCapacity")
+        return "ok"
+
+    def on_retry(attempt, error, wait_seconds):
+        seen.append((attempt, str(error), wait_seconds))
+
+    retry_on_capacity(
+        action,
+        timeout=600,
+        interval=30,
+        on_retry=on_retry,
+        monotonic=clock.monotonic,
+        sleep=clock.sleep,
+    )
+
+    assert len(seen) == 1
+    attempt, msg, wait = seen[0]
+    assert attempt == 1
+    assert "InsufficientInstanceCapacity" in msg
+    assert wait == 30
+
+
+def test_keyboard_interrupt_during_sleep_propagates():
+    class InterruptingClock(FakeClock):
+        def sleep(self, seconds):
+            raise KeyboardInterrupt
+
+    clock = InterruptingClock()
+
+    def action():
+        raise RuntimeError("InsufficientInstanceCapacity")
+
+    with pytest.raises(KeyboardInterrupt):
+        retry_on_capacity(
+            action, timeout=600, interval=30, monotonic=clock.monotonic, sleep=clock.sleep
+        )
+
+
+@pytest.mark.parametrize(
+    "message,expected",
+    [
+        ("InsufficientInstanceCapacity", True),
+        ("Failed to launch instance: (InsufficientInstanceCapacity) ...", True),
+        ("InsufficientFreeAddressesInSubnet", True),
+        ("VcpuLimitExceeded", False),
+        ("InstanceLimitExceeded", False),
+        ("some validation error", False),
+    ],
+)
+def test_is_capacity_error_classification(message, expected):
+    assert is_capacity_error(RuntimeError(message)) is expected