jambu-gpu 0.4.0__tar.gz → 0.4.1__tar.gz

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: jambu-gpu
-Version: 0.4.0
+Version: 0.4.1
 Summary: Provider-agnostic CLI for provisioning and managing temporary GPU runtimes
 Project-URL: Homepage, https://github.com/jambuai/jambu-gpu
 Project-URL: Repository, https://github.com/jambuai/jambu-gpu

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/docs/CONFIG.md

@@ -205,6 +205,7 @@ judged by the sweep's own `--orphan-max-age` and `--stopped-ttl` flags. See
 | --- | --- | --- |
 | `interval` | `60s` | Probe interval while waiting for the runtime. |
 | `boot_timeout` | `30m` | How long the machine may take to reach running, mostly the image pull. Past it the host counts as failed and `setup` tries another (`compute.host_attempts`). |
+| `pull_stall_timeout` | `10m` | How long the image pull may go without progress (no new line from the provider, or only retries of a layer) before the host counts as failed and is replaced, without waiting out `boot_timeout`. Healthy pulls measured live showed a new line at least every 8 minutes; stalled ones never recovered. `off` disables it. |
 | `startup_timeout` | `10m` | Once the machine runs, how long the model server may take to answer. Raise it for very large models. |
 | `path` | `/health` | Health path on the model endpoint. |
 

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/docs/INTEGRATION.md

@@ -117,7 +117,7 @@ gpu env --format dotenv > .env.gpu
 
 | Event | What the app sees | What it does |
 | --- | --- | --- |
-| first `setup` | blocks while a machine is rented, the image pulled and the model downloaded: usually 5–15 minutes. A bad host (stalled image pull, broken network) is destroyed and replaced automatically, each costing up to `health.boot_timeout` (30 min), at most `compute.host_attempts` (3) times | wait; do not put a short timeout on `setup`, it bounds itself |
+| first `setup` | blocks while a machine is rented, the image pulled and the model downloaded: usually 5–15 minutes. A bad host (stalled image pull, broken network) is destroyed and replaced automatically, each costing up to `health.pull_stall_timeout` (10 min) when its pull stalls, or `health.boot_timeout` (30 min) when it crawls, at most `compute.host_attempts` (3) times | wait; do not put a short timeout on `setup`, it bounds itself |
 | idle for `idle_timeout` | the watchdog stops the machine; requests fail with connection refused or a timeout | call `setup` again and retry the request |
 | `setup` on a stopped machine | it is restarted: no new rent, no model download, under a minute for a small model (41 s measured) | wait, then use the **new** `base_url`: the provider may assign a new IP and port on restart |
 | stopped for `stopped_ttl` | the machine is destroyed to stop the storage bill | the next `setup` rents a new one (and a new key) |

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/jambu.example.yaml

@@ -77,6 +77,7 @@ gpu_runtime:
   health:
     interval: 60s
     boot_timeout: 30m
+    pull_stall_timeout: 10m   # a pull with no progress this long: replace the host
     startup_timeout: 10m
     path: /health
 

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/jambu_gpu/__init__.py

@@ -1,5 +1,5 @@
 """gpu: provider-agnostic GPU runtime provisioning and lifecycle CLI."""
 
-__version__ = "0.4.0"
+__version__ = "0.4.1"
 
 __all__ = ["__version__"]

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/jambu_gpu/core/config.py

@@ -230,12 +230,17 @@ class HealthConfig(_Base):
     interval: str = "60s"
     # How long the machine may take to reach running (mostly the image pull).
     boot_timeout: str = "30m"
+    # How long that pull may go without progress before the host counts as
+    # stalled; a stalled pull never resumes, so waiting out boot_timeout
+    # only bills the dead host longer.
+    pull_stall_timeout: str = "10m"
     # How long the model server may then take to answer its health check.
     startup_timeout: str = "10m"
     path: str = "/health"
 
     _v_interval = _duration("interval")
     _v_boot = _duration("boot_timeout")
+    _v_stall = _duration("pull_stall_timeout")
     _v_startup = _duration("startup_timeout")
 
 
@@ -310,6 +315,10 @@ class RuntimeConfig(_Base):
     def boot_timeout_s(self) -> float:
         return parse_duration(self.health.boot_timeout) or 1800.0
 
+    @property
+    def pull_stall_timeout_s(self) -> Optional[float]:
+        return parse_duration(self.health.pull_stall_timeout)
+
     @property
     def startup_timeout_s(self) -> float:
         return parse_duration(self.health.startup_timeout) or 600.0
@@ -579,6 +588,7 @@ gpu_runtime:
   health:
     interval: 60s
     boot_timeout: 30m
+    pull_stall_timeout: 10m
     startup_timeout: 10m
 
   budget:
@@ -628,6 +638,7 @@ lifecycle:
 health:
   interval: 60s
   boot_timeout: 30m
+  pull_stall_timeout: 10m
   startup_timeout: 10m
 
 budget:

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/jambu_gpu/core/engine.py

@@ -61,6 +61,9 @@ _HOST_NETWORK_FAILURE = re.compile(
     r"Temporary failure in name resolution|Network is unreachable"
 )
 
+# A pull stuck retrying a layer: its countdown changes, but it is no progress.
+_PULL_RETRY = re.compile(r"\bretrying\b", re.IGNORECASE)
+
 # Where `gpu logs --source` reads from: the watchdog serves the instance files.
 LOG_SOURCES = ("runtime", "onstart", "watchdog", "provider")
 # Runtime log lines quoted in the error when the model server dies at boot.
@@ -74,6 +77,19 @@ def _noop(_message: str) -> None:
     return None
 
 
+def _pull_progress(message: Optional[str]) -> Optional[str]:
+    """The provider's boot message when it shows the pull moving, else None.
+
+    Vast reports the image pull's latest line (``<layer>: Download complete``,
+    ``<layer>: Pull complete``). A new line means a layer moved; the same line,
+    no line, or a retry countdown means it did not.
+    """
+    text = (message or "").strip()
+    if not text or _PULL_RETRY.search(text):
+        return None
+    return text
+
+
 class Engine:
     def __init__(
         self,
@@ -779,8 +795,13 @@ class Engine:
         instance = state.instance
         assert instance is not None
         boot_timeout = self.config.boot_timeout_s
+        stall_timeout = self.config.pull_stall_timeout_s
         deadline = time.time() + boot_timeout
         announced = False
+        # Live, healthy pulls printed a new line every 1-8 minutes; stalled ones
+        # froze (or retried one layer) until boot_timeout, 30 minutes per host.
+        progress: Optional[str] = None
+        progress_at = time.time()
         while time.time() < deadline:
             status = self.provider.status(instance)
             if not status.exists:
@@ -810,6 +831,18 @@ class Engine:
                 InstanceState.IDLE,
             ) and (have_ports or not ready_ports):
                 return current
+            moved = _pull_progress(status.message)
+            if moved is not None and moved != progress:
+                progress, progress_at = moved, time.time()
+            elif stall_timeout and time.time() - progress_at >= stall_timeout:
+                last = (status.message or "").strip().splitlines()
+                raise HostFailure(
+                    f"instance {instance.id}'s image pull made no progress for "
+                    f"{format_duration(stall_timeout)} (health.pull_stall_timeout)"
+                    + (f"; last: {last[-1][:160]}" if last else ""),
+                    reason=f"image pull stalled for {format_duration(stall_timeout)}",
+                    instance=current,
+                )
             if not announced:
                 announced = True
                 self.echo("  waiting for the machine to come up ...")

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "jambu-gpu"
-version = "0.4.0"
+version = "0.4.1"
 description = "Provider-agnostic CLI for provisioning and managing temporary GPU runtimes"
 readme = "README.md"
 requires-python = ">=3.10"

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/tests/fakes.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import time
-from typing import Optional
+from typing import Callable, Optional
 
 from jambu_gpu.core.errors import CapacityUnavailableError, ProvisioningError
 from jambu_gpu.core.models import (
@@ -66,6 +66,10 @@ class FakeProvider(GPUProvider):
         # reaches running).
         self.failing_hosts: set[str] = set()
         self.host_failure = "failed"
+        # What a "loading" machine reports on its Nth status poll, as Vast
+        # reports the image pull's latest line.
+        self.boot_message: Optional[Callable[[int], str]] = None
+        self.boot_polls = 0
         self.rented_hosts: list[str] = []
         self.searches: list[list[str]] = []
 
@@ -169,6 +173,9 @@ class FakeProvider(GPUProvider):
         if stored is None:
             return InstanceStatus(state=InstanceState.DESTROYED, exists=False)
         message = ""
+        if stored.state is InstanceState.STARTING and self.boot_message is not None:
+            message = self.boot_message(self.boot_polls)
+            self.boot_polls += 1
         if stored.state is InstanceState.FAILED:
             message = (
                 'Error response from daemon: Get "https://registry-1.docker.io/v2/": '

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/tests/test_config.py

@@ -388,3 +388,11 @@ def test_host_attempts_and_boot_timeout_keep_the_instance_compatible(write_confi
     base = write_config().fingerprint()
     assert write_config(compute={"host_attempts": 5}).fingerprint() == base
     assert write_config(health={"boot_timeout": "40m"}).fingerprint() == base
+    assert write_config(health={"pull_stall_timeout": "5m"}).fingerprint() == base
+
+
+def test_pull_stall_timeout_defaults_to_ten_minutes_and_can_be_off(write_config):
+    assert write_config().pull_stall_timeout_s == 10 * 60
+    assert write_config(health={"pull_stall_timeout": "off"}).pull_stall_timeout_s is None
+    with pytest.raises(ConfigError, match="pull_stall_timeout"):
+        write_config(health={"pull_stall_timeout": "soon"})

{jambu_gpu-0.4.0 → jambu_gpu-0.4.1}/tests/test_engine.py

@@ -720,6 +720,78 @@ def test_a_stuck_image_pull_times_out_on_boot_timeout_not_startup_timeout(
     assert health_waits == [420.0]
 
 
+def _with_clock(monkeypatch, start=1_000_000.0):
+    """Make the boot wait's sleeps advance a fake clock: minutes in no time."""
+    clock = [start]
+    monkeypatch.setattr("jambu_gpu.core.engine.time.time", lambda: clock[0])
+    monkeypatch.setattr(
+        "jambu_gpu.core.engine.time.sleep",
+        lambda seconds: clock.__setitem__(0, clock[0] + seconds),
+    )
+    return clock
+
+
+def test_a_frozen_image_pull_is_given_up_long_before_boot_timeout(write_config, monkeypatch):
+    """Live: three hosts froze on one pull line until the 30-minute boot_timeout."""
+    engine, lines, _ = _booting(write_config, monkeypatch)
+    clock = _with_clock(monkeypatch)
+    engine.provider.fail_first_hosts(1, how="loading")
+    engine.provider.boot_message = lambda poll: (
+        "995efd0b979d: Verifying Checksum" if poll < 20 else "067869a57bf4: Pull complete"
+    )
+    started = clock[0]
+
+    state = engine.setup()
+
+    assert state.instance.host_id == "m2"
+    assert "! host m1 failed (image pull stalled for 10m); trying another machine (2/3)" in lines
+    # Two minutes of progress, then ten without: replaced at about 12, not 30.
+    assert 12 * 60 <= clock[0] - started < 13 * 60
+    assert engine.provider.searches[-1] == ["m1"], "the stalled host is not rented again"
+
+
+def test_a_pull_retrying_one_layer_counts_as_stalled(write_config, monkeypatch):
+    engine, lines, _ = _booting(write_config, monkeypatch)
+    clock = _with_clock(monkeypatch)
+    engine.provider.fail_first_hosts(1, how="loading")
+    engine.provider.boot_message = lambda poll: (
+        f"60c68e5ca95a: Retrying in {1 + poll % 3} second{'s' if poll % 3 else ''}"
+    )
+    started = clock[0]
+
+    engine.setup()
+
+    assert "! host m1 failed (image pull stalled for 10m); trying another machine (2/3)" in lines
+    assert 10 * 60 <= clock[0] - started < 11 * 60
+
+
+def test_a_slow_pull_that_keeps_moving_waits_for_boot_timeout(write_config, monkeypatch):
+    """A new layer every 8 minutes is slow, not stuck: only boot_timeout ends it."""
+    engine, lines, _ = _booting(write_config, monkeypatch)
+    clock = _with_clock(monkeypatch)
+    engine.provider.fail_first_hosts(1, how="loading")
+    engine.provider.boot_message = lambda poll: f"layer{poll * 6 // 480}: Pull complete"
+    started = clock[0]
+
+    engine.setup()
+
+    assert "! host m1 failed (image pull timed out after 30m); trying another machine (2/3)" in lines
+    assert clock[0] - started >= 30 * 60
+
+
+def test_pull_stall_detection_can_be_turned_off(write_config, monkeypatch):
+    engine, lines, _ = _booting(
+        write_config, monkeypatch, health={"pull_stall_timeout": "off", "boot_timeout": "20m"}
+    )
+    _with_clock(monkeypatch)
+    engine.provider.fail_first_hosts(1, how="loading")
+    engine.provider.boot_message = lambda poll: "067869a57bf4: Pull complete"
+
+    engine.setup()
+
+    assert "! host m1 failed (image pull timed out after 20m); trying another machine (2/3)" in lines
+
+
 def test_a_model_server_that_dies_is_not_blamed_on_the_host(engine, monkeypatch):
     from jambu_gpu.core.errors import HostFailure, RuntimeStartupError