letify 2.6.1__tar.gz → 2.6.2__tar.gz

{letify-2.6.1 → letify-2.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: letify
-Version: 2.6.1
+Version: 2.6.2
 Summary: Declarations that become infrastructure.
 Project-URL: Homepage, https://github.com/DarkPyonix/letify
 Project-URL: Documentation, https://github.com/DarkPyonix/letify/tree/main/docs

{letify-2.6.1 → letify-2.6.2}/docs/SPEC.md

@@ -1261,7 +1261,7 @@ A `stat` cannot take that snapshot first, because it is on a data loader's hot p
 
 #### When a blob does not arrive <!-- id: project-data-streaming-failures -->
 
-- **A blob that never arrives.** A `data_want` unanswered for `data_wait_timeout` seconds on the account, 600 by default, fails that open with `RuntimeFailure` naming the relative path and the digest. The call is not killed, because the body may handle it; the failure is infrastructure, so a retry of the call is allowed.
+- **A blob that never arrives.** A `data_want` whose blob has not grown for `data_wait_timeout` seconds on the account, 600 by default, fails that open with `RuntimeFailure` naming the relative path, the digest, and how much of the blob had arrived. The call is not killed, because the body may handle it; the failure is infrastructure, so a retry of the call is allowed. The timeout counts time without progress rather than total time, because a blob large enough to need longer than the timeout at the link's rate would otherwise fail however healthy the transfer was: 11.3 GB needs 19.3 MiB/s to land inside 600 s, and a Modal path measured at 45.3 MiB/s for one blob alone did not reach that behind 37 other files. The worker reads the size of the partial it is writing, so progress is the bytes on its own disk and not an estimate.
 - **A sender that stops.** The background sender raising leaves every blob still queued unsendable. letify says so once, `letify: data sending stopped after <n> of <n> files, <n> not sent: <error>`, and tells the runtime, which wakes every reader waiting on one of those blobs with `RuntimeFailure` naming the file and that reason, and fails a reader that arrives later at once. Waiting it out instead costs `data_wait_timeout` once for every blob still queued, which is how a hundred queued shards become an hour of silence.
 - **A client that disconnects mid stream.** The session ends as any lost channel ends it. Every blocked reader is woken with `RuntimeFailure`, the call ends with it, and no partial file is ever placed or read. A `.partial` file is removed when the worker next starts.
 - **A call that finishes before the background transfer.** The transfer is cancelled, both the channel sender and any bucket upload not yet started. A blob already committed to the cache stays and counts as held for the next call, and a partial one is removed. The write-back and the `data_evict` request follow the cancellation, in that order.

{letify-2.6.1 → letify-2.6.2}/letify/__init__.py

@@ -66,7 +66,7 @@ distributed: Final = Host.distributed
 # The enum class stays importable from letify.declare.instance for letify's own use.
 del Host
 
-__version__ = "2.6.1"
+__version__ = "2.6.2"
 
 
 def fetch(tensor):  # type: ignore[no-untyped-def]

{letify-2.6.1 → letify-2.6.2}/letify/protocol/worker.py

@@ -1269,6 +1269,31 @@ def _op_data_failed(request):
     return {"ok": True, "value": True}
 
 
+def _data_received(blobs, digest):
+    """Bytes of this blob that are on the runtime's disk, however they are arriving.
+
+    Spec "When a blob does not arrive": the wait counts time without progress, and this is
+    what progress means. Reads the partial a single put writes and the one a parallel put
+    fills, so neither shape looks stalled while it grows.
+    """
+    total = 0
+    final = _data_file(blobs, digest)
+    directory = os.path.dirname(final)
+    name = os.path.basename(final)
+    try:
+        entries = os.listdir(directory)
+    except OSError:
+        return 0
+    for entry in entries:
+        if entry != name and not entry.startswith(name + "."):
+            continue
+        try:
+            total = max(total, os.stat(os.path.join(directory, entry)).st_size)
+        except OSError:
+            continue
+    return total
+
+
 def _data_want(digests, blocking=False):
     """Ask the local process for these blobs, in this order. Nothing replies to it."""
     if not digests:
@@ -1353,6 +1378,7 @@ def _data_wait(path):
             state["first"] = os.path.basename(path)
         deadline = started + state["wait"]
     _data_want([digest], True)
+    received = _data_received(state["blobs"], digest)
     try:
         with _DATA_READY:
             while path in state["pending"] and state["dir"] in _DATA_CALLS:
@@ -1361,11 +1387,20 @@ def _data_wait(path):
                         "%s was never sent: %s"
                         % (os.path.basename(path), state["failed"])
                     )
+                # Spec "When a blob does not arrive": the wait counts time without
+                # progress, because a blob needing longer than the timeout at the link's
+                # rate would otherwise fail however healthy its transfer was.
+                landed = _data_received(state["blobs"], digest)
+                if landed > received:
+                    received = landed
+                    deadline = time.monotonic() + state["wait"]
                 left = deadline - time.monotonic()
                 if left <= 0:
                     raise _DataFailure(
-                        "the file blob %s of %s did not arrive within %.0f s"
-                        % (digest, os.path.basename(path), state["wait"])
+                        "the file blob %s of %s did not arrive within %.0f s, "
+                        "%.1f MiB of %.1f MiB arrived"
+                        % (digest, os.path.basename(path), state["wait"],
+                           received / 1048576.0, size / 1048576.0)
                     )
                 _DATA_READY.wait(min(left, 0.5))
     finally:

{letify-2.6.1 → letify-2.6.2}/letify-core/Cargo.lock

@@ -138,7 +138,7 @@ dependencies = [
 
 [[package]]
 name = "letify-agent"
-version = "2.6.1"
+version = "2.6.2"
 dependencies = [
  "letify-wire",
  "libloading",
@@ -146,7 +146,7 @@ dependencies = [
 
 [[package]]
 name = "letify-driver"
-version = "2.6.1"
+version = "2.6.2"
 dependencies = [
  "letify-wire",
 ]
@@ -164,7 +164,7 @@ dependencies = [
 
 [[package]]
 name = "letify-wire"
-version = "2.6.1"
+version = "2.6.2"
 
 [[package]]
 name = "libc"

{letify-2.6.1 → letify-2.6.2}/letify-core/Cargo.toml

@@ -8,7 +8,7 @@ resolver = "3"
 members = ["letify-wire", "letify-driver", "letify-agent", "letify-quic"]
 
 [workspace.package]
-version = "2.6.1"
+version = "2.6.2"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/DarkPyonix/letify"

{letify-2.6.1 → letify-2.6.2}/letify-ext/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "letify-status",
-  "version": "2.6.1",
+  "version": "2.6.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "letify-status",
-      "version": "2.6.1",
+      "version": "2.6.2",
       "license": "SEE LICENSE IN LICENSE",
       "devDependencies": {
         "@types/node": "^18.19.0",

{letify-2.6.1 → letify-2.6.2}/letify-ext/package.json

@@ -2,7 +2,7 @@
   "name": "letify-status",
   "displayName": "letify Status",
   "description": "letify quota and GPU activity in the status bar, read from the letify CLI.",
-  "version": "2.6.1",
+  "version": "2.6.2",
   "publisher": "darkpyonix",
   "license": "SEE LICENSE IN LICENSE",
   "repository": {

{letify-2.6.1 → letify-2.6.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "letify"
-version = "2.6.1"
+version = "2.6.2"
 description = "Declarations that become infrastructure."
 readme = "README.md"
 requires-python = ">=3.11"

{letify-2.6.1 → letify-2.6.2}/tests/test_data_streaming.py

@@ -15,6 +15,7 @@ from __future__ import annotations
 
 import collections
 import os
+import pickle
 import re
 import tempfile
 import threading
@@ -1684,3 +1685,75 @@ def test_a_sender_that_stops_fails_the_readers_waiting_on_it(
     assert "was never sent" in str(failure.value), failure.value
     err = capsys.readouterr().err
     assert "data sending stopped after" in err, err
+
+
+def test_a_blob_still_arriving_does_not_time_out(let, cpu, project, monkeypatch) -> None:
+    """Spec "When a blob does not arrive": the timeout counts time without progress.
+
+    KICT's 11.3 GB table failed after 600 s on a Modal path measured at 45.3 MiB/s for a
+    blob on its own. 11.3 GB needs 19.3 MiB/s to land inside 600 s, so a healthy transfer
+    of a blob that large always failed, and the line said 0.0 MiB/s because the rate counts
+    only the blobs that completed.
+    """
+    from letify.store import pathdata
+
+    big = project / "slow.bin"
+    big.write_bytes(b"s" * (6 << 20))
+    monkeypatch.setattr(pathdata, "FIRST_WAVE_MIB", 0.0)
+    monkeypatch.setattr(pathdata, "FIRST_WAVE_FILES", 0)
+    monkeypatch.setattr(pathdata, "CHUNK", 1 << 20)
+    # Each piece lands well inside the wait, and the whole blob takes longer than it.
+    monkeypatch.setattr(pathdata, "WAIT_TIMEOUT_S", 4.0)
+    def piecewise(runtime, pipeline, blobs, digest, local, size, meter):
+        with open(local, "rb") as handle:
+            offset = 0
+            while offset < size:
+                piece = handle.read(1 << 20)
+                if not piece:
+                    break
+                pipeline.send(
+                    {
+                        "op": "data_put",
+                        "dir": blobs,
+                        "digest": digest,
+                        "offset": offset,
+                        "chunk": pickle.PickleBuffer(piece),
+                        "last": offset + len(piece) >= size,
+                    }
+                )
+                offset += len(piece)
+                time.sleep(3.0)
+
+    monkeypatch.setattr(pathdata, "_put", piecewise)
+
+    # retries=0, because a retry finds the blob already there and hides the timeout.
+    @let.function(device=cpu, host=letify.remote, retries=0)
+    def read_it(path) -> int:
+        return len(path.read_bytes())
+
+    assert read_it(path=big) == 6 << 20
+
+
+def test_a_blob_that_never_starts_still_times_out_and_says_how_far_it_got(
+    let, cpu, project, monkeypatch
+) -> None:
+    """Spec "When a blob does not arrive": the timeout still catches a blob that stalls."""
+    from letify.store import pathdata
+
+    big = project / "stalled.bin"
+    big.write_bytes(b"t" * (2 << 20))
+    monkeypatch.setattr(pathdata, "FIRST_WAVE_MIB", 0.0)
+    monkeypatch.setattr(pathdata, "FIRST_WAVE_FILES", 0)
+    monkeypatch.setattr(pathdata, "WAIT_TIMEOUT_S", 4.0)
+    # Nothing is ever sent, so nothing ever grows.
+    monkeypatch.setattr(pathdata.Stream, "_send", lambda self, digest, meter: None)
+
+    @let.function(device=cpu, host=letify.remote)
+    def read_it(path) -> int:
+        return len(path.read_bytes())
+
+    with pytest.raises(RuntimeFailure) as failure:
+        read_it(path=big)
+    message = str(failure.value)
+    assert "did not arrive" in message, message
+    assert "arrived" in message, message