letify 2.6.6__tar.gz → 2.6.7__tar.gz

{letify-2.6.6 → letify-2.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: letify
-Version: 2.6.6
+Version: 2.6.7
 Summary: Declarations that become infrastructure.
 Project-URL: Homepage, https://github.com/DarkPyonix/letify
 Project-URL: Documentation, https://github.com/DarkPyonix/letify/tree/main/docs

{letify-2.6.6 → letify-2.6.7}/docs/SPEC.md

@@ -847,7 +847,20 @@ While the channel waits for `HELLO` each read waits at most 30 s, and a lane rea
 
 Socket buffers are left to the kernel. `SO_SNDBUF` and `SO_RCVBUF` are never set, because setting them turns off the kernel's buffer tuning and the value is capped at `net.core.wmem_max`, 212 KiB by default.
 
-When any step fails, a `tunnel` failure, a connection that does not open, or no `HELLO` within 30 s, the channel prints one line on stderr, `letify: <runtime>: the data channel did not open (<reason>); frames stay on standard input and output`, and keeps using standard input and output. The worker returns to reading standard input when a byte arrives there or its `wait` ends without an authenticated connection, so a request the channel sends over standard input after a failure is answered without waiting for `wait`.
+The tunnel's host name is new when the sandbox is, so a resolver that is asked for it too early can
+answer that it does not exist and keep that answer cached. Resolution is therefore retried for
+`DATA_OPEN_WINDOW`, 30 s, every 3 s, before the channel gives up. Measured on a host where
+`github.com` always resolved and a freshly created tunnel name did not, three sessions in a row fell
+back, and 7.6 GB then went over standard input and output, where one piece held the control path long
+enough for the lease renewal to miss its 60 s timeout and the worker pipe closed. Mornings on the
+same host reached 45 to 90 MiB/s on the same path, so the fallback was the whole difference.
+
+When the window ends without a connection, or a `tunnel` request fails, or no `HELLO` arrives within
+30 s, the channel prints one line on stderr, `letify: <runtime>: the data channel did not open
+(<reason>, host <name>); frames stay on standard input and output, which the control path shares, so
+a large transfer there can starve the lease`, and keeps using standard input and output. The host name
+is named so a reader can resolve it themselves. The channel tries again on the next call rather than
+staying that way, because the name may simply have been too new. The worker returns to reading standard input when a byte arrives there or its `wait` ends without an authenticated connection, so a request the channel sends over standard input after a failure is answered without waiting for `wait`.
 
 A `reexec` request goes over the data connection. The worker replies there and replaces its process, which closes the connection. The channel then sends the worker source over standard input, waits for `HELLO` there, and opens the data channel again.
 

{letify-2.6.6 → letify-2.6.7}/letify/__init__.py

@@ -66,7 +66,7 @@ distributed: Final = Host.distributed
 # The enum class stays importable from letify.declare.instance for letify's own use.
 del Host
 
-__version__ = "2.6.6"
+__version__ = "2.6.7"
 
 
 def fetch(tensor):  # type: ignore[no-untyped-def]

{letify-2.6.6 → letify-2.6.7}/letify/providers/modal.py

@@ -86,6 +86,13 @@ LISTEN_WAIT = 60
 #: Seconds the channel allows for connecting through the tunnel and for the hello after it.
 DATA_CONNECT_TIMEOUT = 30
 
+#: Seconds the channel keeps trying to reach a tunnel whose host name does not resolve,
+#: and the pause between attempts. Spec "Modal data channel": the name is new when the
+#: sandbox is, so a resolver asked too early can answer that it does not exist and cache
+#: that answer.
+DATA_OPEN_WINDOW = 30.0
+DATA_OPEN_PAUSE = 3.0
+
 #: TCP connections the data channel opens when ``data_streams`` is not set, and the most
 #: it accepts. Spec "Parallel data streams".
 DATA_STREAMS = 4
@@ -608,6 +615,23 @@ class TlsStream:
             self._wire.release()
 
 
+def _reach(address: tuple[str, int]) -> socket.socket:
+    """Connect to the tunnel, waiting out a host name the resolver does not have yet.
+
+    Spec "Modal data channel": the name is new when the sandbox is, so a resolver asked
+    too early can answer that it does not exist and keep that answer cached. Anything but
+    a resolution failure is raised at once, because waiting does not help it.
+    """
+    deadline = time.monotonic() + DATA_OPEN_WINDOW
+    while True:
+        try:
+            return socket.create_connection(address, timeout=DATA_CONNECT_TIMEOUT)
+        except socket.gaierror:
+            if time.monotonic() >= deadline:
+                raise
+            time.sleep(DATA_OPEN_PAUSE)
+
+
 class SandboxChannel(FramedChannel):
     """Carries the worker's frames over a sandbox's pipes, through the adapter.
 
@@ -637,6 +661,9 @@ class SandboxChannel(FramedChannel):
         self.sandbox = sandbox
         self.name = name
         self.data_port = data_port
+        #: The tunnel host the last open reached for, named in a failure so a reader can
+        #: resolve it themselves. Spec "Modal data channel".
+        self._tunnel_host: str | None = None
         #: The sandbox's maximum lifetime, so a death at it is named rather than guessed.
         self.lifetime_s = lifetime_s
         self.opened_at = time.monotonic()
@@ -688,9 +715,11 @@ class SandboxChannel(FramedChannel):
             self._connection = self._open_data(self.data_port)
         except (LetifyError, OSError, ValueError, KeyError, TypeError) as exc:
             self._close_socket()
+            where = f", host {self._tunnel_host}" if self._tunnel_host else ""
             print(
-                f"letify: {self.name}: the data channel did not open ({exc}); "
-                "frames stay on standard input and output",
+                f"letify: {self.name}: the data channel did not open ({exc}{where}); "
+                "frames stay on standard input and output, which the control path "
+                "shares, so a large transfer there can starve the lease",
                 file=sys.stderr,
                 flush=True,
             )
@@ -707,6 +736,7 @@ class SandboxChannel(FramedChannel):
         stdio.request(listen, timeout=DATA_CONNECT_TIMEOUT + LISTEN_WAIT, kill=self._kill)
         tunnel = self.adapter.request("tunnel", sandbox=self.sandbox, port=port)
         host = str(tunnel["host"])
+        self._tunnel_host = host
         address = (host, int(tunnel["port"]))
         context = None
         if tunnel.get("tls"):
@@ -717,7 +747,7 @@ class SandboxChannel(FramedChannel):
         def open_lane(index: int) -> None:
             # Spec "Parallel data streams": every lane opens at the same time.
             try:
-                raw = socket.create_connection(address, timeout=DATA_CONNECT_TIMEOUT)
+                raw = _reach(address)
             except BaseException as exc:
                 errors.append(exc)
                 return

{letify-2.6.6 → letify-2.6.7}/letify-core/Cargo.lock

@@ -138,7 +138,7 @@ dependencies = [
 
 [[package]]
 name = "letify-agent"
-version = "2.6.6"
+version = "2.6.7"
 dependencies = [
  "letify-wire",
  "libloading",
@@ -146,7 +146,7 @@ dependencies = [
 
 [[package]]
 name = "letify-driver"
-version = "2.6.6"
+version = "2.6.7"
 dependencies = [
  "letify-wire",
 ]
@@ -164,7 +164,7 @@ dependencies = [
 
 [[package]]
 name = "letify-wire"
-version = "2.6.6"
+version = "2.6.7"
 
 [[package]]
 name = "libc"

{letify-2.6.6 → letify-2.6.7}/letify-core/Cargo.toml

@@ -8,7 +8,7 @@ resolver = "3"
 members = ["letify-wire", "letify-driver", "letify-agent", "letify-quic"]
 
 [workspace.package]
-version = "2.6.6"
+version = "2.6.7"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/DarkPyonix/letify"

{letify-2.6.6 → letify-2.6.7}/letify-ext/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "letify-status",
-  "version": "2.6.6",
+  "version": "2.6.7",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "letify-status",
-      "version": "2.6.6",
+      "version": "2.6.7",
       "license": "SEE LICENSE IN LICENSE",
       "devDependencies": {
         "@types/node": "^18.19.0",

{letify-2.6.6 → letify-2.6.7}/letify-ext/package.json

@@ -2,7 +2,7 @@
   "name": "letify-status",
   "displayName": "letify Status",
   "description": "letify quota and GPU activity in the status bar, read from the letify CLI.",
-  "version": "2.6.6",
+  "version": "2.6.7",
   "publisher": "darkpyonix",
   "license": "SEE LICENSE IN LICENSE",
   "repository": {

{letify-2.6.6 → letify-2.6.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "letify"
-version = "2.6.6"
+version = "2.6.7"
 description = "Declarations that become infrastructure."
 readme = "README.md"
 requires-python = ">=3.11"

{letify-2.6.6 → letify-2.6.7}/tests/test_providers.py

@@ -2941,3 +2941,55 @@ def test_a_sandbox_that_reached_its_lifetime_says_so(config_file) -> None:
     assert "3600" in lifetime_detail(age_s=3599.5, lifetime_s=3600)
     assert "raise timeout" in lifetime_detail(age_s=3601.0, lifetime_s=3600)
     assert lifetime_detail(age_s=120.0, lifetime_s=3600) is None
+
+
+def test_a_tunnel_name_that_is_not_in_dns_yet_is_waited_for(
+    isolated_home, fake_modal, monkeypatch
+) -> None:
+    """Spec "Modal data channel": the host name is new when the sandbox is.
+
+    KICT lost three sessions in a row to a resolver that answered NXDOMAIN for a freshly
+    created tunnel name while github.com resolved every time, and 7.6 GB then went over
+    standard input and output until the lease renewal missed its timeout.
+    """
+    import socket as socketmod
+
+    from letify.providers import modal as modalmod
+
+    attempts: list[float] = []
+    real = socketmod.create_connection
+
+    def refuse_then_open(address, timeout=None, *args, **kwargs):
+        attempts.append(0.0)
+        if len(attempts) < 3:
+            raise socketmod.gaierror(-3, "Temporary failure in name resolution")
+        return real(address, timeout, *args, **kwargs)
+
+    monkeypatch.setattr(modalmod.socket, "create_connection", refuse_then_open)
+    monkeypatch.setattr(modalmod, "DATA_OPEN_PAUSE", 0.01)
+    provider = provider_of(Modal, "m")
+    runtime = modal_runtime(provider)
+    channel = provider.open_channel(runtime)
+    try:
+        channel.start()
+        assert channel.call(sum, ([1, 2, 3],), {})[0] == 6
+        assert len(attempts) >= 3, f"gave up after {len(attempts)} attempt(s)"
+    finally:
+        channel.close()
+        provider.stop(runtime)
+
+
+def test_a_data_channel_failure_names_the_host(isolated_home, fake_modal, capfd) -> None:
+    """Spec "Modal data channel": the host name is named so a reader can resolve it."""
+    fake_modal.fail("tunnel")
+    provider = provider_of(Modal, "m")
+    runtime = modal_runtime(provider)
+    channel = provider.open_channel(runtime)
+    try:
+        channel.start()
+        assert channel.call(sum, ([1, 2, 3],), {})[0] == 6
+        err = capfd.readouterr().err
+        assert "the control path shares" in err, err
+    finally:
+        channel.close()
+        provider.stop(runtime)