letify 2.8.4__tar.gz → 2.8.5__tar.gz

{letify-2.8.4 → letify-2.8.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: letify
-Version: 2.8.4
+Version: 2.8.5
 Summary: Declarations that become infrastructure.
 Project-URL: Homepage, https://github.com/DarkPyonix/letify
 Project-URL: Documentation, https://github.com/DarkPyonix/letify/tree/main/docs

{letify-2.8.4 → letify-2.8.5}/docs/SPEC.md

@@ -777,7 +777,7 @@ The protocol is one JSON object per line. letify sends `{"id": <int>, "op": <nam
 | Op | Fields | Value |
 |---|---|---|
 | `hello` | none | `{"modal": <installed Modal version>}` |
-| `create` | `app`, `args`, `packages`, `gpu`, `timeout`, `idle_timeout`, `volumes`, `ports` | `{"sandbox": <id>}`. Runs `app` as an ephemeral app on first use, builds `debian_slim` with `packages` installed, and starts `args` in a sandbox. Each port in `ports` is exposed with Modal `encrypted_ports` |
+| `create` | `app`, `args`, `packages`, `gpu`, `timeout`, `idle_timeout`, `memory`, `volumes`, `ports` | `{"sandbox": <id>}`. Runs `app` as an ephemeral app on first use, builds `debian_slim` with `packages` installed, and starts `args` in a sandbox. Each port in `ports` is exposed with Modal `encrypted_ports` |
 | `tunnel` | `sandbox`, `port` | `{"host": <text>, "port": <int>, "tls": <bool>}`. The address that reaches `port` inside the sandbox, from `Sandbox.tunnels()`. `tls` is true when the connection has to be made with TLS, which an encrypted port needs |
 | `write` | `sandbox`, `data` | `null`. `data` is base64 of at most 1 MiB. Writes the decoded bytes to the sandbox's standard input and drains it. Modal refuses a write that would buffer more than 2 MiB, so the channel splits a larger frame into `write` requests of 1 MiB, in order |
 | `read_until` | `sandbox`, `prefixes` | `{"lines": [...], "eof": <bool>}`. The sandbox's stdout lines up to and including the first that starts with one of `prefixes`, or every line left when the stream ends |
@@ -812,8 +812,12 @@ Modal also bounds a sandbox that nothing terminates. `create` passes `idle_timeo
 
 It passes `timeout`, the entry option `timeout`, as the sandbox's maximum lifetime, and the default is the longest Modal accepts, 86400 s, measured as accepted. It is not a shorter number letify picked, for the reason The declaration gives for `timeout` having no default: a deadline letify invented would end a two hour training run at whatever hour it guessed. The default was 3600 s, exactly one hour, and it ended two Modal sessions of a real training run at one hour each with nothing saying why, which is that sentence happening. An account that wants a hard cap sets `timeout` itself.
 
+It passes `memory`, the entry option `memory_gib` converted to MiB, as the sandbox's memory request, and passes nothing when the account does not set it, so Modal's default applies. A training process whose RAM grows past what the sandbox gets is killed with SIGKILL, which the channel sees as a worker that stopped without replying; an account whose runs need more sets `memory_gib`, for example `memory_gib = 64`. A value that is not a positive number raises `ConfigError` naming `memory_gib`.
+
 A sandbox that reaches that lifetime takes its worker with it, so the channel sees a worker that stopped without replying. Where the session's age is within a minute of the lifetime letify asked for, the failure says so instead of listing causes: `the Modal sandbox reached the <seconds> s lifetime letify asked for; raise timeout on the account to run longer`.
 
+Otherwise a Modal worker that stopped without replying is followed by one `poll` of its sandbox from a second adapter process, waiting at most 30 s for the exit code, and the failure names what the poll found instead of listing causes: `the Modal sandbox exited with code <n>` for a sandbox that ended, with `, which is SIGKILL, most often the container running out of memory` added for 137, and `the Modal sandbox is still running, so the worker process died inside it` for one still running. A poll that fails or times out leaves the message as it was. The message is what tells an out-of-memory kill from a preemption, which a training run that died twice in one day could not.
+
 ### Modal data channel <!-- id: modal-data-channel -->
 
 > Once the worker has said hello over the sandbox's standard input and output, the channel moves its frames to a TCP connection through a Modal encrypted port. Standard input and output stay the control path and the fallback.

{letify-2.8.4 → letify-2.8.5}/letify/__init__.py

@@ -68,7 +68,7 @@ distributed: Final = Host.distributed
 # The enum class stays importable from letify.declare.instance for letify's own use.
 del Host
 
-__version__ = "2.8.4"
+__version__ = "2.8.5"
 
 
 def fetch(tensor):  # type: ignore[no-untyped-def]

{letify-2.8.4 → letify-2.8.5}/letify/providers/modal.py

@@ -31,7 +31,7 @@ import tempfile
 import threading
 import time
 import weakref
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from typing import IO, TYPE_CHECKING, Any
 
 from ..config import ProviderConfig
@@ -133,6 +133,22 @@ def lifetime_detail(*, age_s: float, lifetime_s: int) -> str | None:
     )
 
 
+def exit_cause(found: tuple[bool, int | None] | None) -> str | None:
+    """What a poll of a sandbox whose worker stopped says, or None when it is unknown.
+
+    Spec "Modal": the exit code tells an out of memory kill from a preemption.
+    """
+    if found is None:
+        return None
+    ended, code = found
+    if not ended:
+        return "the Modal sandbox is still running, so the worker process died inside it"
+    text = f"the Modal sandbox exited with code {code}"
+    if code == 137:
+        text += ", which is SIGKILL, most often the container running out of memory"
+    return text
+
+
 class VolumePathMissing(RuntimeFailure):
     """The adapter answered that a volume path does not exist."""
 
@@ -419,6 +435,21 @@ class Modal(Provider):
         value = self.config.option("idle_timeout", IDLE_TIMEOUT)
         return max(1, int(value))
 
+    @property
+    def sandbox_memory_mib(self) -> int | None:
+        """The sandbox's memory request in MiB from ``memory_gib``, or None for Modal's own.
+
+        Spec "Modal": a training process past what the sandbox gets is killed with SIGKILL.
+        """
+        value = self.config.option("memory_gib")
+        if value is None:
+            return None
+        if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
+            raise ConfigError(
+                f"{self.alias}: memory_gib has to be a positive number of GiB, not {value!r}"
+            )
+        return int(value * 1024)
+
     def adapter(self) -> Adapter:
         """The adapter acting as this account, started on its first request."""
         if self._adapter is None or self._adapter.closed:
@@ -484,6 +515,7 @@ class Modal(Provider):
             gpu=self.wire_name(runtime.instance) or None,
             timeout=self.sandbox_timeout_s,
             idle_timeout=self.sandbox_idle_timeout_s,
+            memory=self.sandbox_memory_mib,
             ports=[] if data_port is None else [data_port],
         )
         sandbox = str(created["sandbox"])
@@ -491,17 +523,14 @@ class Modal(Provider):
         return SandboxChannel(
             adapter, sandbox, name=runtime.name, data_port=data_port, streams=streams,
             lifetime_s=self.sandbox_timeout_s,
+            probe=lambda: self.poll_sandbox(sandbox),
         )
 
-    def runtime_state(self, runtime: Runtime) -> RuntimeState:
-        """Poll the sandbox from a second adapter process, never the one carrying the call.
+    def poll_sandbox(self, sandbox: str) -> tuple[bool, int | None] | None:
+        """Poll a sandbox once from a second adapter: ``(ended, exit code)``, or None if unknown.
 
-        Spec "Lifetime", the lease. The adapter carrying the call may be blocked in a
-        request, so a short-lived one asks. Any failure answers unknown.
+        A short-lived adapter asks, because the one carrying the call may be blocked.
         """
-        sandbox = self._sandboxes.get(runtime.name)
-        if sandbox is None:
-            return RuntimeState()
         other = Adapter.for_account(self.alias)
         # A poll that hangs ends the second adapter, which makes the request raise.
         limit = threading.Timer(POLL_LIMIT, other.abort)
@@ -510,11 +539,26 @@ class Modal(Provider):
         try:
             code = other.request("poll", sandbox=sandbox)
         except (LetifyError, ValueError, OSError):
-            return RuntimeState()
+            return None
         finally:
             limit.cancel()
             other.close()
-        if code is None:
+        return (code is not None, code)
+
+    def runtime_state(self, runtime: Runtime) -> RuntimeState:
+        """Poll the sandbox from a second adapter process, never the one carrying the call.
+
+        Spec "Lifetime", the lease. The adapter carrying the call may be blocked in a
+        request, so a short-lived one asks. Any failure answers unknown.
+        """
+        sandbox = self._sandboxes.get(runtime.name)
+        if sandbox is None:
+            return RuntimeState()
+        found = self.poll_sandbox(sandbox)
+        if found is None:
+            return RuntimeState()
+        ended, code = found
+        if not ended:
             return RuntimeState("running")
         return RuntimeState(
             "ended", f"the Modal sandbox ended (exit code {code}) while the call was running"
@@ -686,8 +730,11 @@ class SandboxChannel(FramedChannel):
         data_port: int | None = None,
         streams: int = 1,
         lifetime_s: int = 0,
+        probe: Callable[[], tuple[bool, int | None] | None] | None = None,
     ):
         self.adapter = adapter
+        #: Polls the sandbox from a second adapter, see ``Modal.poll_sandbox``.
+        self.probe = probe
         self.sandbox = sandbox
         self.name = name
         self.data_port = data_port
@@ -716,6 +763,7 @@ class SandboxChannel(FramedChannel):
             wire.chunks_readinto(self._read_chunks),
             self._emit,
             death_detail=self._raw_text,
+            death_cause=self._exit_cause,
         )
         self._connection = self._stdio
         self._send_worker()
@@ -730,6 +778,17 @@ class SandboxChannel(FramedChannel):
         )
         return f"{said}\n{text}" if said else text
 
+    def _exit_cause(self) -> str | None:
+        """What one poll of the sandbox found, or None to keep the usual causes.
+
+        Spec "Modal": a lifetime death is already named by ``_raw_text``, so it is not polled.
+        """
+        if self.probe is None or lifetime_detail(
+            age_s=time.monotonic() - self.opened_at, lifetime_s=self.lifetime_s
+        ):
+            return None
+        return exit_cause(self.probe())
+
     def _send_worker(self) -> None:
         # After a reexec the data connection is gone, so the source goes back over stdio.
         if self._connection is not self._stdio:
@@ -821,7 +880,12 @@ class SandboxChannel(FramedChannel):
                 )
                 carrier.timeout = DATA_CONNECT_TIMEOUT
             connection = Connection(
-                self.name, carrier.send, carrier.recv_into, self._emit, death_detail=self._raw_text
+                self.name,
+                carrier.send,
+                carrier.recv_into,
+                self._emit,
+                death_detail=self._raw_text,
+                death_cause=self._exit_cause,
             )
             # The connect timeout still applies, so a missing hello fails here.
             connection.wait_hello()

{letify-2.8.4 → letify-2.8.5}/letify/providers/modal_adapter.py

@@ -109,6 +109,7 @@ class Adapter:
             gpu=request.get("gpu") or None,
             timeout=int(request["timeout"]),
             idle_timeout=int(request["idle_timeout"]) if request.get("idle_timeout") else None,
+            memory=int(request["memory"]) if request.get("memory") else None,
             volumes={
                 str(path): modal.Volume.from_name(str(name), create_if_missing=True)
                 for path, name in (request.get("volumes") or {}).items()

{letify-2.8.4 → letify-2.8.5}/letify/runtime/channel.py

@@ -258,6 +258,7 @@ class Connection:
         emit: Callable[[str, bytes], None],
         *,
         death_detail: Callable[[], str] = lambda: "",
+        death_cause: Callable[[], str | None] = lambda: None,
         poll_fd: int | None = None,
     ):
         self.name = name
@@ -276,6 +277,8 @@ class Connection:
         self._receiver = wire.Receiver(readinto)
         self._emit = emit
         self._death_detail = death_detail
+        #: Names what ended the worker when the provider can tell, replacing the usual causes.
+        self._death_cause = death_cause
         #: Guards ``_reading`` and wakes waiters when a reply, a hello or a failure arrives.
         self._turn = threading.Condition()
         self._reading = False
@@ -383,10 +386,16 @@ class Connection:
             return
         if message is None:
             detail = self._death_detail()
+            cause = self._death_cause()
+            causes = (
+                f": {cause}."
+                if cause
+                else ". The usual causes are an out of memory kill, a preempted session, "
+                "or a crash below Python."
+            )
             message = (
                 f"{self.name}: the worker stopped without replying, so the process died "
-                f"before it finished. The usual causes are an out of memory kill, a "
-                f"preempted session, or a crash below Python.\n--- last remote output ---\n"
+                f"before it finished{causes}\n--- last remote output ---\n"
                 f"{self._tail.text()[-2000:]}{detail[-2000:]}"
             )
         with self._turn:

{letify-2.8.4 → letify-2.8.5}/letify-core/Cargo.lock

@@ -138,7 +138,7 @@ dependencies = [
 
 [[package]]
 name = "letify-agent"
-version = "2.8.4"
+version = "2.8.5"
 dependencies = [
  "letify-wire",
  "libloading",
@@ -146,7 +146,7 @@ dependencies = [
 
 [[package]]
 name = "letify-driver"
-version = "2.8.4"
+version = "2.8.5"
 dependencies = [
  "letify-wire",
 ]
@@ -164,7 +164,7 @@ dependencies = [
 
 [[package]]
 name = "letify-wire"
-version = "2.8.4"
+version = "2.8.5"
 
 [[package]]
 name = "libc"

{letify-2.8.4 → letify-2.8.5}/letify-core/Cargo.toml

@@ -8,7 +8,7 @@ resolver = "3"
 members = ["letify-wire", "letify-driver", "letify-agent", "letify-quic"]
 
 [workspace.package]
-version = "2.8.4"
+version = "2.8.5"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/DarkPyonix/letify"

{letify-2.8.4 → letify-2.8.5}/letify-ext/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "letify-status",
-  "version": "2.8.4",
+  "version": "2.8.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "letify-status",
-      "version": "2.8.4",
+      "version": "2.8.5",
       "license": "SEE LICENSE IN LICENSE",
       "devDependencies": {
         "@types/node": "^18.19.0",

{letify-2.8.4 → letify-2.8.5}/letify-ext/package.json

@@ -2,7 +2,7 @@
   "name": "letify-status",
   "displayName": "letify Status",
   "description": "letify quota and GPU activity in the status bar, read from the letify CLI.",
-  "version": "2.8.4",
+  "version": "2.8.5",
   "publisher": "darkpyonix",
   "license": "SEE LICENSE IN LICENSE",
   "repository": {

{letify-2.8.4 → letify-2.8.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "letify"
-version = "2.8.4"
+version = "2.8.5"
 description = "Declarations that become infrastructure."
 readme = "README.md"
 requires-python = ">=3.11"

{letify-2.8.4 → letify-2.8.5}/tests/test_providers.py

@@ -2368,6 +2368,39 @@ def test_a_modal_sandbox_is_created_with_a_lifetime_and_an_idle_limit(
         provider.stop(runtime)
 
 
+def test_a_modal_sandbox_requests_the_memory_the_account_sets(isolated_home, fake_modal) -> None:
+    # Spec "Modal": memory_gib becomes the sandbox's memory request in MiB.
+    provider = provider_of(Modal, "m", memory_gib=64)
+    runtime = modal_runtime(provider)
+    provider.open_channel(runtime)
+    try:
+        [created] = fake_modal.requests("create")
+        assert created["memory"] == 64 * 1024
+    finally:
+        provider.stop(runtime)
+
+
+def test_a_modal_sandbox_requests_no_memory_when_the_account_sets_none(
+    isolated_home, fake_modal
+) -> None:
+    provider = provider_of(Modal, "m")
+    runtime = modal_runtime(provider)
+    provider.open_channel(runtime)
+    try:
+        [created] = fake_modal.requests("create")
+        assert created.get("memory") is None
+    finally:
+        provider.stop(runtime)
+
+
+@pytest.mark.parametrize("value", [0, -2, "lots", True])
+def test_a_memory_gib_that_is_not_a_positive_number_is_refused(config_file, value) -> None:
+    from letify.errors import ConfigError
+
+    with pytest.raises(ConfigError, match="memory_gib"):
+        _ = provider_of(Modal, "lab", memory_gib=value).sandbox_memory_mib
+
+
 # -- Modal data channel ----------------------------------------------------------
 
 
@@ -3205,3 +3238,73 @@ def test_stopping_a_colab_orphan_unassigns_its_endpoint(
     assert provider.stop_orphan("gpu-a") is True
     assert cli.unassigned == ["gpu-a"]
     assert provider.orphans() == []
+
+
+# -- Modal: a worker that stopped names what the sandbox did (spec "Modal") -----------
+
+
+def _died_modal_call(fake_modal, provider, *, ended: int | None) -> str:
+    """Kill the worker behind a Modal channel, make a call, and return the failure text."""
+    import os
+    import signal
+    import time
+
+    from letify.errors import ProtocolError
+
+    runtime = modal_runtime(provider)
+    channel = provider.open_channel(runtime)
+    channel.start()
+    [pid] = fake_modal.sandbox_pids()
+    if ended is not None:
+        fake_modal.end_sandbox("sb-1", ended)
+    os.kill(pid, signal.SIGKILL)
+    try:
+        with pytest.raises(ProtocolError) as caught:
+            channel.call(time.sleep, (600,), {}, timeout=20)
+    finally:
+        channel.close()
+        provider.stop(runtime)
+    return str(caught.value)
+
+
+def test_a_worker_killed_with_exit_137_names_the_out_of_memory_kill(
+    isolated_home, fake_modal
+) -> None:
+    text = _died_modal_call(fake_modal, provider_of(Modal, "m"), ended=137)
+    assert "the Modal sandbox exited with code 137, which is SIGKILL" in text
+    assert "most often the container running out of memory" in text
+    assert "The usual causes" not in text
+
+
+def test_a_worker_whose_sandbox_exited_with_1_names_the_code_only(
+    isolated_home, fake_modal
+) -> None:
+    text = _died_modal_call(fake_modal, provider_of(Modal, "m"), ended=1)
+    assert "the Modal sandbox exited with code 1" in text
+    assert "SIGKILL" not in text
+    assert "The usual causes" not in text
+
+
+def test_a_worker_that_died_inside_a_running_sandbox_says_so(isolated_home, fake_modal) -> None:
+    text = _died_modal_call(fake_modal, provider_of(Modal, "m"), ended=None)
+    assert "the Modal sandbox is still running, so the worker process died inside it" in text
+    assert "The usual causes" not in text
+
+
+def test_a_failed_poll_leaves_the_worker_death_message_as_it_was(
+    isolated_home, fake_modal
+) -> None:
+    fake_modal.fail("poll")
+    text = _died_modal_call(fake_modal, provider_of(Modal, "m"), ended=137)
+    assert "The usual causes are an out of memory kill" in text
+    assert "the Modal sandbox exited" not in text
+
+
+def test_a_sandbox_at_its_lifetime_keeps_the_lifetime_message_without_a_poll(
+    isolated_home, fake_modal
+) -> None:
+    provider = provider_of(Modal, "m", timeout=1)
+    text = _died_modal_call(fake_modal, provider, ended=137)
+    assert "lifetime letify asked for" in text
+    assert "exited with code" not in text
+    assert fake_modal.requests("poll") == []