letify 2.4.4__tar.gz → 2.4.5__tar.gz

{letify-2.4.4 → letify-2.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: letify
-Version: 2.4.4
+Version: 2.4.5
 Summary: Declarations that become infrastructure.
 Project-URL: Homepage, https://github.com/DarkPyonix/letify
 Project-URL: Documentation, https://github.com/DarkPyonix/letify/tree/main/docs

{letify-2.4.4 → letify-2.4.5}/letify/__init__.py

@@ -64,7 +64,7 @@ distributed: Final = Host.distributed
 # The enum class stays importable from letify.declare.instance for letify's own use.
 del Host
 
-__version__ = "2.4.4"
+__version__ = "2.4.5"
 
 
 def fetch(tensor):  # type: ignore[no-untyped-def]

{letify-2.4.4 → letify-2.4.5}/letify/declare/function.py

@@ -65,8 +65,11 @@ class Function(Generic[R]):
         self.data_order = data_order
         self.data_first_wave = data_first_wave
         self.is_async = inspect.iscoroutinefunction(fn)
-        #: Spec "Replicas": one Instance per replica, in the order the list named them.
-        self.replicas = _replicas(device, self.host)
+        #: Spec "Replicas": one Instance per replica, in the order the list named them,
+        #: each carrying the declared placement so a provider is not told it is local.
+        self.replicas = tuple(
+            instance._placed(self.host) for instance in _replicas(device, self.host)
+        )
         self.device = self._place(self.replicas[0] if self.replicas else device)
         update_wrapper(self, fn)
 
@@ -139,18 +142,21 @@ class Function(Generic[R]):
     def _run(self, args: tuple, kwargs: dict) -> Any:
         launcher = self.launcher
         if self.host is Host.distributed:
-            # Spec "Replicas": the placement is declared and the runner is not written
-            # yet. Running the first replica alone would bill one runtime for a
-            # declaration that asked for several, without saying so.
-            raise UnsupportedMode(
-                f"host=letify.distributed declares {len(self.replicas)} replicas and "
-                f"letify cannot run them yet, so the call is refused rather than run on "
-                f"one of them. Use host=letify.remote with a single device until it is "
-                f"finished."
-            )
+            return self._run_replicas(args, kwargs)
         instance = launcher.resolve(self.device)
         if instance.placement is Host.local:
             return self._run_here(instance, args, kwargs)
+        return self._run_one(instance, args, kwargs)
+
+    def _run_one(
+        self, instance: Any, args: tuple, kwargs: dict, replica: dict | None = None
+    ) -> Any:
+        """Run the body on one runtime, retrying on an infrastructure failure.
+
+        One replica of a distributed call runs through here too, so a replica gets the
+        same retries and the same reporting as a call that named a single device.
+        """
+        launcher = self.launcher
         last: Exception | None = None
 
         for attempt in range(self.retries + 1):
@@ -166,6 +172,7 @@ class Function(Generic[R]):
                     timeout=self.timeout,
                     data_order=self.data_order,
                     data_first_wave=self.data_first_wave,
+                    replica=_with_rate(replica, runtime),
                 )
             except (RuntimeFailure, ProtocolError) as failure:
                 # The session misbehaved rather than the user's code, so this runtime
@@ -200,6 +207,40 @@ class Function(Generic[R]):
 
         raise RuntimeLost(str(last))  # pragma: no cover
 
+    def _run_replicas(self, args: tuple, kwargs: dict) -> list:
+        """Run the body once per replica, and answer with their values in rank order.
+
+        Spec "Replicas": each replica is told which one it is, the call waits for all of
+        them, and a replica that fails fails the call, as a single remote call's failure
+        does. The replicas run at the same time, because running them in turn would make
+        a reduction between them impossible.
+        """
+        import threading
+
+        world = len(self.replicas)
+        values: list = [None] * world
+        failures: list[BaseException] = []
+
+        def run(rank: int, instance: Any) -> None:
+            try:
+                values[rank] = self._run_one(
+                    instance, args, kwargs, {"rank": rank, "world_size": world}
+                )
+            except BaseException as exc:  # reported after every replica has stopped
+                failures.append(exc)
+
+        threads = [
+            threading.Thread(target=run, args=(rank, instance), daemon=True)
+            for rank, instance in enumerate(self.replicas)
+        ]
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+        if failures:
+            raise failures[0]
+        return values
+
     def _run_here(self, instance: Any, args: tuple, kwargs: dict) -> Any:
         """Run the body in this process with its PyTorch operators on the runtime's device.
 
@@ -261,6 +302,23 @@ def _report_deferred(delta: Any) -> None:
         )
 
 
+def _with_rate(replica: dict | None, runtime: Any) -> dict | None:
+    """The replica's numbers plus the rate its own link measured, for ``local_steps``.
+
+    Spec "How often to synchronize": the rate is the quantity only letify has, and it is
+    this replica's link rather than the job's, so it travels with the call.
+    """
+    if replica is None:
+        return None
+    rate = None
+    try:
+        link = runtime.provider.link(runtime)
+        rate = getattr(link, "upload_bps", None)
+    except Exception:
+        rate = None
+    return {**replica, "rate_bps": rate}
+
+
 def _replicas(device: Any, host: Host) -> tuple[Instance, ...]:
     """The replicas a declaration names, or empty when it names a single device.
 

{letify-2.4.4 → letify-2.4.5}/letify/protocol/worker.py

@@ -316,6 +316,7 @@ def _reply(stream, outcome):
 
 def _op_call(request):
     data = request.pop("data", None)
+    _letify_replica(request.pop("replica", None))
     modules_dir = request.pop("modules_dir", None)
     if modules_dir:
         modules_dir = os.path.expanduser(modules_dir)
@@ -349,6 +350,25 @@ def _op_call(request):
 _DATA_WRITTEN = {}
 
 
+def _letify_replica(replica):
+    """Tell the letify in this runtime which replica this call is, or that it is none.
+
+    Spec "Replicas": the body reads its rank through letify, which is installed here by
+    the environment, so the numbers are set on the module rather than passed through the
+    user's signature.
+    """
+    try:
+        from letify.declare import replicas as _replicas
+    except Exception:
+        return
+    if replica is None:
+        _replicas._WHO.rank = _replicas._WHO.world_size = _replicas._WHO.rate_bps = None
+        return
+    _replicas._WHO.rank = replica.get("rank")
+    _replicas._WHO.world_size = replica.get("world_size")
+    _replicas._WHO.rate_bps = replica.get("rate_bps")
+
+
 def _data_hash_file(path):
     """The file digest of a file on the runtime: blake3 when present, else blake2b."""
     hasher = _data_hasher()

{letify-2.4.4 → letify-2.4.5}/letify/runtime/bootstrap.py

@@ -133,7 +133,7 @@ def project_dir(workspace_root: str, env: Env) -> str:
 #: Reads the cards a runtime has, without nvidia-smi, which a Kaggle image lacks on PATH.
 #: Spec "Kaggle", the accelerators.
 CARD_SOURCE = (
-    "import glob\n"
+    "import glob, os\n"
     "__letify_cards = []\n"
     "for _letify_path in sorted(glob.glob('/proc/driver/nvidia/gpus/*/information')):\n"
     "    try:\n"
@@ -144,7 +144,12 @@ CARD_SOURCE = (
     "                    break\n"
     "    except OSError:\n"
     "        pass\n"
-    "__letify_value__ = __letify_cards\n"
+    # /proc can list the right cards on a session whose driver cannot be used, and the
+    # first .to('cuda') then raises minutes into the call. Spec "Kaggle".
+    "__letify_value__ = {\n"
+    "    'cards': __letify_cards,\n"
+    "    'driver': os.path.exists('/dev/nvidiactl'),\n"
+    "}\n"
 )
 
 

{letify-2.4.4 → letify-2.4.5}/letify/runtime/session.py

@@ -269,6 +269,7 @@ class Runtime:
         timeout: float | None = None,
         data_order: Any = None,
         data_first_wave: int | None = None,
+        replica: dict[str, Any] | None = None,
     ) -> tuple[Any, str]:
         """Run one declared function inside this runtime.
 
@@ -283,7 +284,9 @@ class Runtime:
                 fn, args, kwargs, timeout=timeout, modules=self.env.ship_modules
             )
         args, kwargs = self._externalize(args, kwargs)
-        return self._call_with_data(fn, args, kwargs, timeout, data_order, data_first_wave)
+        return self._call_with_data(
+            fn, args, kwargs, timeout, data_order, data_first_wave, replica
+        )
 
     def _call_with_data(
         self,
@@ -293,6 +296,7 @@ class Runtime:
         timeout: float | None,
         data_order: Any = None,
         data_first_wave: int | None = None,
+        replica: dict[str, Any] | None = None,
     ) -> tuple[Any, str]:
         """Pickle a call, place the first wave of its data, and send the rest beside it.
 
@@ -317,6 +321,10 @@ class Runtime:
                     ship_digests.add(digest)
         head, buffers = protocol.dumps_call_parts(fn, args, kwargs, data=collector)
         request: dict[str, Any] = {"op": "call", "payload": head, "buffers": buffers}
+        if replica is not None:
+            # Spec "Replicas": the body reads its rank on the runtime, not here, so the
+            # numbers travel with the call rather than being set on a client thread.
+            request["replica"] = replica
         blobs = f"{root}/data/blobs"
         stream: pathdata.Stream | None = None
         if self.env.ship_modules:
@@ -708,16 +716,25 @@ class Runtime:
             return
         fragment, count = expected
         try:
-            cards = self.eval(bootstrap.CARD_SOURCE, timeout=120)
+            found = self.eval(bootstrap.CARD_SOURCE, timeout=120)
         except Exception:
             # Reading /proc is not worth failing a session over when it cannot be read.
             return
-        cards = [str(card) for card in cards or ()]
+        found = found if isinstance(found, dict) else {"cards": found, "driver": True}
+        cards = [str(card) for card in found.get("cards") or ()]
+        driver = bool(found.get("driver"))
         matched = [card for card in cards if fragment.casefold() in card.casefold()]
-        if len(matched) == count and len(cards) == count:
+        if len(matched) == count and len(cards) == count and driver:
             return
         asked = getattr(instance, "accelerator", None) or fragment
         arrived = ", ".join(cards) if cards else "no GPU at all"
+        if cards and not driver:
+            # Spec "Kaggle": the cards are right and the driver cannot be used, which the
+            # first .to("cuda") would have reported minutes into the call instead.
+            arrived = (
+                f"{arrived}, and no usable driver: /dev/nvidiactl is not there, so CUDA "
+                f"cannot start"
+            )
         message = (
             f"{self.name}: {self.provider.alias} was asked for {count} x {asked} and the "
             f"session has {arrived}. A declaration says where a function runs, so the "

{letify-2.4.4 → letify-2.4.5}/letify-core/Cargo.lock

@@ -138,7 +138,7 @@ dependencies = [
 
 [[package]]
 name = "letify-agent"
-version = "2.4.4"
+version = "2.4.5"
 dependencies = [
  "letify-wire",
  "libloading",
@@ -146,7 +146,7 @@ dependencies = [
 
 [[package]]
 name = "letify-driver"
-version = "2.4.4"
+version = "2.4.5"
 dependencies = [
  "letify-wire",
 ]
@@ -164,7 +164,7 @@ dependencies = [
 
 [[package]]
 name = "letify-wire"
-version = "2.4.4"
+version = "2.4.5"
 
 [[package]]
 name = "libc"

{letify-2.4.4 → letify-2.4.5}/letify-core/Cargo.toml

@@ -8,7 +8,7 @@ resolver = "3"
 members = ["letify-wire", "letify-driver", "letify-agent", "letify-quic"]
 
 [workspace.package]
-version = "2.4.4"
+version = "2.4.5"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/DarkPyonix/letify"

{letify-2.4.4 → letify-2.4.5}/letify-ext/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "letify-status",
-  "version": "2.4.4",
+  "version": "2.4.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "letify-status",
-      "version": "2.4.4",
+      "version": "2.4.5",
       "license": "SEE LICENSE IN LICENSE",
       "devDependencies": {
         "@types/node": "^18.19.0",

{letify-2.4.4 → letify-2.4.5}/letify-ext/package.json

@@ -2,7 +2,7 @@
   "name": "letify-status",
   "displayName": "letify Status",
   "description": "letify quota and GPU activity in the status bar, read from the letify CLI.",
-  "version": "2.4.4",
+  "version": "2.4.5",
   "publisher": "darkpyonix",
   "license": "SEE LICENSE IN LICENSE",
   "repository": {

{letify-2.4.4 → letify-2.4.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "letify"
-version = "2.4.4"
+version = "2.4.5"
 description = "Declarations that become infrastructure."
 readme = "README.md"
 requires-python = ">=3.11"

{letify-2.4.4 → letify-2.4.5}/tests/test_distributed.py

@@ -127,23 +127,73 @@ def test_local_steps_refuses_a_link_that_was_never_measured() -> None:
     assert "probe" in str(failure.value)
 
 
-def test_calling_a_distributed_declaration_says_it_is_not_finished(launcher_from) -> None:
-    """Spec "Replicas": the placement is declared and the runner is not written yet.
+def test_every_replica_runs_the_body_and_knows_which_one_it_is(launcher_from) -> None:
+    """Spec "Replicas": the same body once per device, and the values come back in rank order."""
+    let = launcher_from('[a]\nkind = "local"\n\n[b]\nkind = "local"\n')
+
+    @let.function(
+        device=[let.providers.a.CPU, let.providers.b.CPU], host=letify.distributed
+    )
+    def who() -> tuple[int, int]:
+        import letify as inner
+
+        return inner.rank(), inner.world_size()
+
+    assert who() == [(0, 2), (1, 2)]
+
+
+def test_one_replica_still_comes_back_as_a_list(launcher_from) -> None:
+    """Spec "Replicas": a single instance is one replica, so the shape does not change."""
+    let = launcher_from('[a]\nkind = "local"\n')
+
+    @let.function(device=let.providers.a.CPU, host=letify.distributed)
+    def who() -> tuple[int, int]:
+        import letify as inner
+
+        return inner.rank(), inner.world_size()
+
+    assert who() == [(0, 1)]
+
 
-    Accepting the declaration and then running on the first replica alone would be worse
-    than refusing: the user asked for several and would be billed for one, silently.
+def test_a_replica_that_fails_fails_the_call(launcher_from) -> None:
+    """Spec "Replicas": a replica that cannot run fails the call, as one remote call does."""
+    let = launcher_from('[a]\nkind = "local"\n\n[b]\nkind = "local"\n')
+
+    @let.function(
+        device=[let.providers.a.CPU, let.providers.b.CPU], host=letify.distributed
+    )
+    def breaks() -> int:
+        import letify as inner
+
+        if inner.rank() == 1:
+            raise ValueError("the second replica refused")
+        return 0
+
+    with pytest.raises(letify.RemoteError) as failure:
+        breaks()
+    assert "the second replica refused" in str(failure.value)
+
+
+def test_a_replica_does_not_need_torch_on_this_machine(launcher_from, no_module) -> None:
+    """Spec "Replicas": the placement is not local, so nothing asks for PyTorch here.
+
+    The replicas were handed to the provider without the declared placement folded in,
+    so their placement read as the default, local, and `check_mode` asked for PyTorch on
+    the client. It passed on a machine that has torch and failed on CI, which does not.
     """
     let = launcher_from('[a]\nkind = "local"\n\n[b]\nkind = "local"\n')
 
     @let.function(
         device=[let.providers.a.CPU, let.providers.b.CPU], host=letify.distributed
     )
-    def body() -> int:
-        return 1
+    def who() -> int:
+        import letify as inner
 
-    with pytest.raises(letify.UnsupportedMode) as failure:
-        body()
-    message = str(failure.value)
-    assert "cannot run them yet" in message
-    assert "2 replicas" in message
-    assert "letify.remote" in message
+        return inner.rank()
+
+    no_module("torch")
+    assert who() == [0, 1]
+    assert [instance.placement for instance in who.replicas] == [
+        letify.distributed,
+        letify.distributed,
+    ]

{letify-2.4.4 → letify-2.4.5}/tests/test_kaggle.py

@@ -1451,3 +1451,48 @@ def test_a_refused_run_says_how_long_ago_the_last_session_was_made(
     assert "too soon" in detail
     path.write_text("", encoding="utf-8")
     assert refusal_detail("kaggle_a") == ""
+
+
+def test_a_session_with_the_cards_and_no_driver_is_refused() -> None:
+    """Spec "Kaggle": the cards can be listed on a session whose driver cannot be used.
+
+    A live T4 session passed the card check and then raised "Found no NVIDIA driver on
+    your system" at the first .to("cuda"), minutes into the call.
+    """
+    from letify.errors import InsufficientDevices
+    from letify.runtime.session import Runtime
+
+    provider = kaggle_provider()
+    session = object.__new__(Runtime)
+    session.name = "letify-t4-fc1bf0"
+    session.provider = provider
+    session.instance = type("I", (), {"gpu": "T4", "accelerator": "T4"})()
+    answer = {"cards": ["Tesla T4", "Tesla T4"], "driver": False}
+    monkey = lambda self, source, timeout=None: answer  # noqa: E731
+    Runtime.eval, original = monkey, Runtime.eval
+    try:
+        with pytest.raises(InsufficientDevices) as failure:
+            session.check_devices()
+    finally:
+        Runtime.eval = original
+    message = str(failure.value)
+    assert "driver" in message
+    assert "Tesla T4" in message
+
+
+def test_a_session_with_the_cards_and_a_driver_passes() -> None:
+    """Spec: both halves have to hold, and then the session is used."""
+    from letify.runtime.session import Runtime
+
+    provider = kaggle_provider()
+    session = object.__new__(Runtime)
+    session.name = "letify-t4-ok"
+    session.provider = provider
+    session.instance = type("I", (), {"gpu": "T4", "accelerator": "T4"})()
+    answer = {"cards": ["Tesla T4", "Tesla T4"], "driver": True}
+    monkey = lambda self, source, timeout=None: answer  # noqa: E731
+    Runtime.eval, original = monkey, Runtime.eval
+    try:
+        session.check_devices()
+    finally:
+        Runtime.eval = original