letify 2.5.3__tar.gz → 2.5.4__tar.gz

{letify-2.5.3 → letify-2.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: letify
-Version: 2.5.3
+Version: 2.5.4
 Summary: Declarations that become infrastructure.
 Project-URL: Homepage, https://github.com/DarkPyonix/letify
 Project-URL: Documentation, https://github.com/DarkPyonix/letify/tree/main/docs

{letify-2.5.3 → letify-2.5.4}/docs/SPEC.md

@@ -1239,6 +1239,7 @@ A `stat` cannot take that snapshot first, because it is on a data loader's hot p
 #### When a blob does not arrive <!-- id: project-data-streaming-failures -->
 
 - **A blob that never arrives.** A `data_want` unanswered for `data_wait_timeout` seconds on the account, 600 by default, fails that open with `RuntimeFailure` naming the relative path and the digest. The call is not killed, because the body may handle it; the failure is infrastructure, so a retry of the call is allowed.
+- **A sender that stops.** The background sender raising leaves every blob still queued unsendable. letify says so once, `letify: data sending stopped after <n> of <n> files, <n> not sent: <error>`, and tells the runtime, which wakes every reader waiting on one of those blobs with `RuntimeFailure` naming the file and that reason, and fails a reader that arrives later at once. Waiting it out instead costs `data_wait_timeout` once for every blob still queued, which is how a hundred queued shards become an hour of silence.
 - **A client that disconnects mid stream.** The session ends as any lost channel ends it. Every blocked reader is woken with `RuntimeFailure`, the call ends with it, and no partial file is ever placed or read. A `.partial` file is removed when the worker next starts.
 - **A call that finishes before the background transfer.** The transfer is cancelled, both the channel sender and any bucket upload not yet started. A blob already committed to the cache stays and counts as held for the next call, and a partial one is removed. The write-back and the `data_evict` request follow the cancellation, in that order.
 - **A bucket-backed ephemeral provider.** A pull that fails is retried twice with the same downscoped headers, then a third time with headers the local process derives again, and a pull that still fails fails the open with `RuntimeFailure`. A `data_want` for a digest not yet uploaded moves that object to the front of the upload queue, and the worker pulls it when the local process sends its `data_pull`.

{letify-2.5.3 → letify-2.5.4}/letify/__init__.py

@@ -66,7 +66,7 @@ distributed: Final = Host.distributed
 # The enum class stays importable from letify.declare.instance for letify's own use.
 del Host
 
-__version__ = "2.5.3"
+__version__ = "2.5.4"
 
 
 def fetch(tensor):  # type: ignore[no-untyped-def]

{letify-2.5.3 → letify-2.5.4}/letify/protocol/worker.py

@@ -364,6 +364,8 @@ def _op_call(request):
         import shutil
         _data_unregister(state)
         _data_check_cache(data, state["placed"])
+        _DATA_FAILED.pop(data["dir"], None)
+        _DATA_FAILED.pop("", None)
         _DATA_POLL_SEEN.pop(data["dir"], None)
         _DATA_POLL_SENT.pop(data["dir"], None)
         shutil.rmtree(data["dir"], ignore_errors=True)
@@ -1047,6 +1049,9 @@ def _data_register(data):
         "linked": 0,
         "order": {},
         "arrived": 0,
+        # Spec "When a blob does not arrive": set when the local sender stops, so a
+        # reader fails at once with its reason rather than waiting out the timeout.
+        "failed": _DATA_FAILED.get(data["dir"]) or _DATA_FAILED.get(""),
         "wait": float(data.get("wait") or 600.0),
         "observe": bool(data.get("observe", True)),
         "prefetch": int(data.get("prefetch") or 64),
@@ -1238,6 +1243,32 @@ def _data_arrived(blobs, digest):
         _DATA_READY.notify_all()
 
 
+#: Reasons a sender stopped, by call directory, for a call that registers afterwards.
+_DATA_FAILED = {}
+
+
+def _op_data_failed(request):
+    """The local sender stopped, so the blobs it had left will never arrive.
+
+    Spec "When a blob does not arrive": every reader waiting on one of them is woken with
+    the reason, and a reader that arrives later fails on it at once, because waiting out
+    the per blob timeout costs that timeout once for every blob still queued.
+    """
+    reason = request.get("reason") or "the local process stopped sending"
+    where = request.get("dir") or ""
+    with _DATA_READY:
+        # Remembered by directory as well as set on the live call, because the sender can
+        # stop before the call request has been read, and the call then registers after
+        # this and would otherwise inherit nothing. Spec "When a blob does not arrive".
+        _DATA_FAILED[where] = reason
+        for state in _DATA_CALLS.values():
+            if where and state["dir"] != where:
+                continue
+            state["failed"] = reason
+        _DATA_READY.notify_all()
+    return {"ok": True, "value": True}
+
+
 def _data_want(digests, blocking=False):
     """Ask the local process for these blobs, in this order. Nothing replies to it."""
     if not digests:
@@ -1304,6 +1335,10 @@ def _data_wait(path):
         entry = state["pending"].get(path)
         if entry is None:
             return
+        if state["failed"]:
+            raise _DataFailure(
+                "%s was never sent: %s" % (os.path.basename(path), state["failed"])
+            )
         digest, size = entry
         index = state["order"].get(digest)
         if index is not None and index > state["arrived"] + _DATA_MISS_AHEAD:
@@ -1321,6 +1356,11 @@ def _data_wait(path):
     try:
         with _DATA_READY:
             while path in state["pending"] and state["dir"] in _DATA_CALLS:
+                if state["failed"]:
+                    raise _DataFailure(
+                        "%s was never sent: %s"
+                        % (os.path.basename(path), state["failed"])
+                    )
                 left = deadline - time.monotonic()
                 if left <= 0:
                     raise _DataFailure(
@@ -2289,6 +2329,7 @@ _OPS = {
     "data_put": _op_data_put,
     "data_begin": _op_data_begin,
     "data_drop": _op_data_drop,
+    "data_failed": _op_data_failed,
     "data_put_at": _op_data_put_at,
     "data_seal": _op_data_seal,
     "data_pull": _op_data_pull,
@@ -2314,7 +2355,9 @@ _OPS = {
 }
 
 #: Answered by the frame reader at once, so they work while a call runs.
-_LIGHT = ("stat", "lease", "reduce_peers")
+# ``data_failed`` is light because it must not queue behind the call it is about: the call
+# is blocked on a blob the sender will never send, so a queued answer would deadlock.
+_LIGHT = ("stat", "lease", "reduce_peers", "data_failed")
 
 #: Answered by several replies, the last one marked. These send their own replies.
 _STREAMING = ("data_stream",)

{letify-2.5.3 → letify-2.5.4}/letify/runtime/session.py

@@ -419,7 +419,7 @@ class Runtime:
             needed = [d for d in plan.missing if d in ship_digests and d not in wave]
             if needed:
                 wave = needed + wave
-        stream = pathdata.Stream(self, blobs, plan)
+        stream = pathdata.Stream(self, blobs, plan, collector.call_dir)
         if not plan.missing:
             # Nothing to stream: the runtime holds every file, so the call places them all
             # before the body starts. It needs no manifest, no patch and no observation.

{letify-2.5.3 → letify-2.5.4}/letify/store/pathdata.py

@@ -23,7 +23,7 @@ from importlib import import_module
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
-from ..errors import ConfigError, RemoteError, RuntimeFailure
+from ..errors import ConfigError, ProtocolError, RemoteError, RuntimeFailure
 
 if TYPE_CHECKING:
     from ..providers.base import Provider
@@ -608,10 +608,12 @@ class Stream:
     order after the call starts comes from the runtime rather than from the prediction.
     """
 
-    def __init__(self, runtime: Runtime, blobs: str, plan: Plan) -> None:
+    def __init__(self, runtime: Runtime, blobs: str, plan: Plan, call_dir: str = "") -> None:
         self.runtime = runtime
         self.blobs = blobs
         self.plan = plan
+        #: The call this stream feeds, so a stopped sender fails that call's readers only.
+        self.call_dir = call_dir
         #: Digests whose send finished. ``_sent`` holds the ones it began, so a blob the
         #: sender was carrying when the call ended is in one and not the other.
         self.done: set[str] = set()
@@ -662,13 +664,35 @@ class Stream:
                     break
             # The pieces still in flight are answered before the stream is released.
             self._flush()
-        except Exception:
-            # The call owns the failure: a blob that never arrives fails the read that
-            # waits for it, with the message spec "When a blob does not arrive" names.
-            pass
+        except Exception as exc:
+            # Spec "When a blob does not arrive": the sender stopping leaves every blob
+            # still queued unsendable, so it is said once and the readers waiting on one
+            # are failed with the reason. Waiting it out costs the per blob timeout once
+            # for each, which is how 131 queued shards became a silent hour.
+            if not self._cancelled.is_set():
+                self._stopped(exc)
         finally:
             self.during_seconds = time.monotonic() - started
 
+    def _stopped(self, exc: BaseException) -> None:
+        """Say the sender stopped, and tell the runtime so its readers stop waiting."""
+        from .. import install
+
+        left = self.dropped
+        reason = f"{type(exc).__name__}: {exc}"
+        install.log(
+            f"data sending stopped after {len(self.done)} of "
+            f"{len(self.plan.missing)} files, {len(left)} not sent: {reason}"
+        )
+        try:
+            _worker(
+                self.runtime,
+                {"op": "data_failed", "dir": self.call_dir, "reason": reason},
+            )
+        except (RuntimeFailure, RemoteError, ProtocolError):
+            # The runtime is gone, which fails every reader on its own.
+            pass
+
     def _pump(self) -> bool:
         """Send the next blob of the queue. False once nothing is left to send."""
         self.reorder()

{letify-2.5.3 → letify-2.5.4}/letify-core/Cargo.lock

@@ -138,7 +138,7 @@ dependencies = [
 
 [[package]]
 name = "letify-agent"
-version = "2.5.3"
+version = "2.5.4"
 dependencies = [
  "letify-wire",
  "libloading",
@@ -146,7 +146,7 @@ dependencies = [
 
 [[package]]
 name = "letify-driver"
-version = "2.5.3"
+version = "2.5.4"
 dependencies = [
  "letify-wire",
 ]
@@ -164,7 +164,7 @@ dependencies = [
 
 [[package]]
 name = "letify-wire"
-version = "2.5.3"
+version = "2.5.4"
 
 [[package]]
 name = "libc"

{letify-2.5.3 → letify-2.5.4}/letify-core/Cargo.toml

@@ -8,7 +8,7 @@ resolver = "3"
 members = ["letify-wire", "letify-driver", "letify-agent", "letify-quic"]
 
 [workspace.package]
-version = "2.5.3"
+version = "2.5.4"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/DarkPyonix/letify"

{letify-2.5.3 → letify-2.5.4}/letify-ext/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "letify-status",
-  "version": "2.5.3",
+  "version": "2.5.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "letify-status",
-      "version": "2.5.3",
+      "version": "2.5.4",
       "license": "SEE LICENSE IN LICENSE",
       "devDependencies": {
         "@types/node": "^18.19.0",

{letify-2.5.3 → letify-2.5.4}/letify-ext/package.json

@@ -2,7 +2,7 @@
   "name": "letify-status",
   "displayName": "letify Status",
   "description": "letify quota and GPU activity in the status bar, read from the letify CLI.",
-  "version": "2.5.3",
+  "version": "2.5.4",
   "publisher": "darkpyonix",
   "license": "SEE LICENSE IN LICENSE",
   "repository": {

{letify-2.5.3 → letify-2.5.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "letify"
-version = "2.5.3"
+version = "2.5.4"
 description = "Declarations that become infrastructure."
 readme = "README.md"
 requires-python = ">=3.11"

{letify-2.5.3 → letify-2.5.4}/tests/test_data_streaming.py

@@ -1641,3 +1641,46 @@ def test_a_cancelled_send_leaves_no_partial_in_the_blob_cache(
     root = Path.home() / ".letify" / "runtime" / "data" / "blobs"
     leftovers = [p for p in root.rglob("*") if p.is_file() and ".letify-" in p.name]
     assert leftovers == [], leftovers
+
+
+def test_a_sender_that_stops_fails_the_readers_waiting_on_it(
+    let, cpu, project, capsys, monkeypatch
+) -> None:
+    """Spec "When a blob does not arrive": waiting it out costs the timeout per blob.
+
+    KICT sent 126.9 GB as 132 shards over the first wave budget. The sender raised on the
+    second one, _run swallowed it, and the body's readers waited in silence: 540 s and
+    counting on one shard, with 131 behind it.
+    """
+    from letify.store import pathdata
+
+    first = project / "a.bin"
+    second = project / "b.bin"
+    first.write_bytes(b"a" * (1 << 20))
+    second.write_bytes(b"b" * (1 << 20))
+    monkeypatch.setattr(pathdata, "FIRST_WAVE_MIB", 0.0)
+    monkeypatch.setattr(pathdata, "FIRST_WAVE_FILES", 0)
+
+    real_send = pathdata.Stream._send
+    sent: list[str] = []
+
+    def one_then_raise(self, digest, meter):
+        if sent:
+            raise RuntimeFailure("the transfer stream failed")
+        sent.append(digest)
+        return real_send(self, digest, meter)
+
+    monkeypatch.setattr(pathdata.Stream, "_send", one_then_raise)
+
+    @let.function(device=cpu, host=letify.remote)
+    def read_both(one, two) -> int:
+        return len(one.read_bytes()) + len(two.read_bytes())
+
+    started = time.monotonic()
+    with pytest.raises(RuntimeFailure) as failure:
+        read_both(one=first, two=second)
+    elapsed = time.monotonic() - started
+    assert elapsed < 60, f"waited {elapsed:.0f} s, so it waited out a timeout"
+    assert "was never sent" in str(failure.value), failure.value
+    err = capsys.readouterr().err
+    assert "data sending stopped after" in err, err