letify 1.2.4__tar.gz → 2.0.0__tar.gz

{letify-1.2.4 → letify-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: letify
-Version: 1.2.4
+Version: 2.0.0
 Summary: Declarations that become infrastructure.
 Project-URL: Homepage, https://github.com/DarkPyonix/letify
 Project-URL: Documentation, https://github.com/DarkPyonix/letify/tree/main/docs

{letify-1.2.4 → letify-2.0.0}/letify/__init__.py

@@ -62,7 +62,7 @@ remote: Final = Host.remote
 # The enum class stays importable from letify.declare.instance for letify's own use.
 del Host
 
-__version__ = "1.2.4"
+__version__ = "2.0.0"
 
 
 def fetch(tensor):  # type: ignore[no-untyped-def]

{letify-1.2.4 → letify-2.0.0}/letify/install.py

@@ -626,6 +626,64 @@ def remote_tailcat_source() -> str:
     )
 
 
+def remote_quic_source(version: str) -> str:
+    """Prepare ``letify-quic`` on the remote from the letify wheel of this version.
+
+    Spec "QUIC over a punched UDP pair": the binary changes with every letify release, so
+    a hash pinned in this source cannot describe the build that is about to be made. The
+    remote therefore fetches the wheel for its own platform at exactly this version, which
+    PyPI verifies, and takes the binary out of it.
+    """
+    import inspect
+
+    return (
+        inspect.getsource(_remote_quic)
+        + f"\nbinary = _remote_quic({version!r})\n"
+        + "import json\nprint('LETIFY-QUIC ' + json.dumps(binary))\n"
+    )
+
+
+def _remote_quic(version):
+    """Take letify-quic out of the letify wheel for this machine, standard library only."""
+    import json
+    import os
+    import subprocess
+    import sys
+    import tempfile
+    import zipfile
+    from pathlib import Path
+
+    inside = "letify/remoting/lib/letify-quic"
+    target = Path(tempfile.gettempdir()) / f"letify-quic-{version}"
+    if target.is_file() and os.access(target, os.X_OK):
+        return str(target)
+    directory = tempfile.mkdtemp(prefix="letify-quic-")
+    done = subprocess.run(
+        [
+            sys.executable, "-m", "pip", "download", f"letify=={version}",
+            "--no-deps", "--only-binary", ":all:", "-d", directory,
+        ],
+        capture_output=True,
+        text=True,
+        timeout=600,
+    )
+    if done.returncode != 0:
+        raise RuntimeError(
+            "could not download the letify wheel for letify-quic: "
+            + (done.stderr or done.stdout).strip()[-400:]
+        )
+    wheels = sorted(Path(directory).glob("letify-*.whl"))
+    if not wheels:
+        raise RuntimeError("pip downloaded no letify wheel")
+    with zipfile.ZipFile(wheels[0]) as archive:
+        if inside not in archive.namelist():
+            raise RuntimeError(f"{wheels[0].name} carries no {inside}")
+        target.write_bytes(archive.read(inside))
+    target.chmod(0o755)
+    json.dumps(str(target))
+    return str(target)
+
+
 def _remote_tailcat(version, release, pins):
     """Install a verified Linux tailcat using only the runtime's standard library."""
     import hashlib

{letify-1.2.4 → letify-2.0.0}/letify/protocol/worker.py

@@ -694,6 +694,79 @@ def _op_data_put(request):
     return {"ok": True, "value": None}
 
 
+def _parallel_partial(root, digest):
+    """Where parallel senders write. One name per digest, not per process or thread."""
+    return "%s%s.parallel" % (_data_file(root, digest), _DATA_PARTIAL)
+
+
+def _op_data_begin(request):
+    """Make the file parallel senders will fill, of the size it will end up.
+
+    Spec "Several connections at once". A placement starts here rather than at the write
+    of offset zero, because with several connections offset zero is just another range and
+    may arrive last: truncating on it would throw away what the others had written.
+    """
+    partial = _parallel_partial(request["dir"], request["digest"])
+    os.makedirs(os.path.dirname(partial), exist_ok=True)
+    with open(partial, "wb") as handle:
+        handle.truncate(int(request["size"]))
+    return {"ok": True, "value": None}
+
+
+def _op_data_put_at(request):
+    """Write one piece of a file blob at its offset, for several senders at once.
+
+    No hashing: a hash over a stream cannot be computed out of order, so ``data_seal``
+    does it by reading the finished file.
+    """
+    partial = _parallel_partial(request["dir"], request["digest"])
+    chunk = request.pop("chunk")
+    handle = open(partial, "r+b")
+    try:
+        handle.seek(int(request["offset"]))
+        handle.write(chunk)
+    finally:
+        handle.close()
+    chunk = None
+    return {"ok": True, "value": None}
+
+
+def _op_data_seal(request):
+    """Hash a file assembled by parallel senders, then commit it as ``data_put`` does."""
+    digest = request["digest"]
+    root = request["dir"]
+    partial = _parallel_partial(root, digest)
+    final = _data_file(root, digest)
+    size = request.get("size")
+    try:
+        found = os.path.getsize(partial)
+    except OSError as exc:
+        raise ValueError("no pieces of file blob %s arrived: %s" % (digest, exc))
+    if size is not None and found != int(size):
+        raise ValueError(
+            "file blob %s is short: %d of %d bytes arrived" % (digest, found, int(size))
+        )
+    hasher = _data_hasher()
+    if hasher is not None:
+        with open(partial, "rb") as handle:
+            while True:
+                piece = handle.read(1 << 22)
+                if not piece:
+                    break
+                hasher.update(piece)
+        got = hasher.hexdigest(length=16)
+        if got != digest:
+            os.remove(partial)
+            raise ValueError("file blob %s arrived with digest %s" % (digest, got))
+    try:
+        os.chmod(partial, 0o444)
+    except OSError:
+        pass
+    os.replace(partial, final)
+    _data_arrived(root, digest)
+    return {"ok": True, "value": None}
+
+
 def _op_data_pull(request):
     """Download file blobs from the bucket, eight at a time, then forget the headers."""
     import urllib.request
@@ -1875,6 +1948,9 @@ _OPS = {
     "have": _op_have,
     "data_have": _op_data_have,
     "data_put": _op_data_put,
+    "data_begin": _op_data_begin,
+    "data_put_at": _op_data_put_at,
+    "data_seal": _op_data_seal,
     "data_pull": _op_data_pull,
     "data_evict": _op_data_evict,
     "data_cache": _op_data_cache,

{letify-1.2.4 → letify-2.0.0}/letify/providers/base.py

@@ -260,6 +260,42 @@ class Provider(abc.ABC):
             return "ephemeral"
         return self.default_persistence
 
+    @property
+    def transfer_streams(self) -> int:
+        """Connections a large blob is split across, from ``transfer_streams``.
+
+        Spec "Several connections at once": one TCP connection over a long round trip
+        carries a fraction of what the path will carry, and independent connections add
+        up. One keeps the single stream.
+        """
+        from ..store.pathdata import STREAMS
+
+        value = self.config.option("transfer_streams")
+        return max(1, int(value)) if isinstance(value, (int, float)) else STREAMS
+
+    @property
+    def transfer_parallel_rtt_ms(self) -> int:
+        """The round trip above which a blob is split, from ``transfer_parallel_rtt_ms``.
+
+        Below it one stream already carries everything the link will carry, so the extra
+        connections are pure cost. Spec "Several connections at once".
+        """
+        from ..store.pathdata import PARALLEL_RTT_MS
+
+        value = self.config.option("transfer_parallel_rtt_ms")
+        return max(0, int(value)) if isinstance(value, (int, float)) else PARALLEL_RTT_MS
+
+    @property
+    def transfer_parallel_mib(self) -> int:
+        """The size above which a blob is split, from ``transfer_parallel_mib``.
+
+        Below it, opening more connections costs more than it saves.
+        """
+        from ..store.pathdata import PARALLEL_MIB
+
+        value = self.config.option("transfer_parallel_mib")
+        return max(0, int(value)) if isinstance(value, (int, float)) else PARALLEL_MIB
+
     @property
     def persistent(self) -> bool:
         return self.persistence == "persistent"

{letify-1.2.4 → letify-2.0.0}/letify/providers/colab.py

@@ -323,30 +323,17 @@ class Colab(Shell):
     def rendezvous(self, runtime: Runtime | None = None) -> Rendezvous | None:
         if runtime is None:
             return None
-        from ..transport.rendezvous import ColabRendezvous
+        from ..transport.rendezvous import ColabRendezvous, prepare_tailcat
 
         name = runtime.name
 
-        def prepare_tailcat() -> str | None:
-            from .. import install
-
-            if not (
-                install.find("tailcat", link_cache=False) or self.config.option("tailcat_binary")
-            ):
-                return None
-            output = self._exec(name, install.remote_tailcat_source(), 180)
-            for line in output.splitlines():
-                if line.startswith("LETIFY-TAILCAT "):
-                    return json.loads(line[len("LETIFY-TAILCAT ") :])
-            raise RuntimeFailure(
-                "tailcat installation gave no verified binary path",
-                stderr=output.strip(),
-            )
+        def run(source: str, timeout: float) -> str:
+            return self._exec(name, source, timeout)
 
         return ColabRendezvous(
-            lambda source, timeout: self._exec(name, source, timeout),
+            run,
             self._public_key(),
-            prepare_tailcat=prepare_tailcat,
+            prepare_tailcat=prepare_tailcat(run, self.config.option("tailcat_binary")),
         )
 
     def fallback(self, runtime: Runtime | None = None) -> Callable[[], Link] | None:

{letify-1.2.4 → letify-2.0.0}/letify/providers/kaggle.py

@@ -25,6 +25,7 @@ import binascii
 import json
 import math
 import re
+import textwrap
 import time
 import urllib.error
 import urllib.parse
@@ -929,6 +930,48 @@ class KaggleChannel(FramedChannel):
             process.kill()
 
 
+def _capture_stdout(source: str) -> str:
+    """``source`` with its standard output captured into ``__letify_value__``.
+
+    A ``from __future__`` import has to be the first statement of the module, so those
+    lines are hoisted above the wrapper rather than indented under it. Without that the
+    rendezvous program, which begins with one, fails to compile and the punch reports that
+    the remote half gave no answer.
+    """
+    lines = source.splitlines(keepends=True)
+    future = [line for line in lines if line.lstrip().startswith("from __future__")]
+    rest = [line for line in lines if not line.lstrip().startswith("from __future__")]
+    return (
+        "".join(future)
+        + "import contextlib as _c, io as _io\n"
+        + "__letify_buffer__ = _io.StringIO()\n"
+        + "with _c.redirect_stdout(__letify_buffer__):\n"
+        + textwrap.indent("".join(rest), "    ")
+        + "\n__letify_value__ = __letify_buffer__.getvalue()\n"
+    )
+
+
+def _key_path():
+    """Where letify keeps the key pair a Kaggle rendezvous authorizes."""
+    from pathlib import Path as _Path
+
+    return _Path.home() / ".ssh" / "id_letify"
+
+
+def _public_key() -> str | None:
+    """letify's own public key, so the rendezvous can authorize it on the machine."""
+    try:
+        return _key_path().with_suffix(".pub").read_text(encoding="utf-8").strip() or None
+    except OSError:
+        return None
+
+
+def _private_key() -> str | None:
+    """The private half, which the SSH command over a punched link logs in with."""
+    path = _key_path()
+    return str(path) if path.exists() else None
+
+
 class Kaggle(Provider):
     """One Kaggle account."""
 
@@ -1005,6 +1048,8 @@ class Kaggle(Provider):
         #: runs its programs in. The notebook is this runtime's own, created in
         #: ``open_channel`` and deleted in ``stop``.
         self._kernels: dict[str, tuple[Session, str, KaggleChannel, int, int, str | None]] = {}
+        #: The link chosen over the bridge, per runtime, closed with the session.
+        self._links: dict[str, Any] = {}
         #: Monotonic time of the last confirmed-live cookie check, or None before the first.
         self._cookie_checked_at: float | None = None
 
@@ -1039,6 +1084,126 @@ class Kaggle(Provider):
         self._cookie_checked_at = now
         return cookie
 
+    # -- an SSH link over the kernel ----------------------------------------------
+
+    def rendezvous_over(self, channel: Any) -> Any:
+        """The bridge as a rendezvous: it runs one program on the machine and reports.
+
+        Spec "Kaggle runtimes", An SSH link over the kernel. The kernel is how letify
+        reaches the machine; it does not have to be how the session is carried.
+        """
+        from ..transport.rendezvous import CommandRendezvous, prepare_tailcat
+
+        bridge = channel
+
+        class KaggleRendezvous(CommandRendezvous):
+            """``eval`` on the open bridge, which is a standard library program on the VM."""
+
+            def extras(self) -> dict[str, Any]:
+                # The VM has no key of letify's yet, and no SSH server running.
+                extra: dict[str, Any] = {"start_sshd": True}
+                public = _public_key()
+                if public:
+                    extra["authorized_key"] = public
+                return extra
+
+            def run_python(self, source: str, timeout: float | None) -> str:
+                # The rendezvous program prints its answer, and ``eval`` returns what the
+                # source left in __letify_value__, not what it printed: the print goes out
+                # as a STDOUT frame and never reaches the caller. So the source runs with
+                # its standard output captured into that name.
+                captured = _capture_stdout(source)
+                value, _ = bridge.request(
+                    {"op": "eval", "source": captured}, timeout=timeout
+                )
+                return value if isinstance(value, str) else str(value or "")
+
+        rendezvous = KaggleRendezvous()
+        # A Kaggle image ships no tailcat, so the request has to carry an installed path.
+        rendezvous._prepare_tailcat = prepare_tailcat(
+            rendezvous.run_python, self.config.option("tailcat_binary")
+        )
+        return rendezvous
+
+    def target_over(self, channel: Any) -> Any:
+        """What the strategies need, with no address because there is none to dial."""
+        from ..transport import nat
+        from ..transport.strategies import Target
+
+        user = self.config.option("user")
+        return Target(
+            alias=self.alias,
+            rendezvous=self.rendezvous_over(channel),
+            remote_python=self.remote_python,
+            stun=nat.DEFAULT_STUN,
+            workspace=self.workspace_root,
+            # The rendezvous authorized this key for root on the VM, so the link has to
+            # offer it and name that user. Without them the VM closes the banner exchange.
+            key=_private_key(),
+            user=user if isinstance(user, str) and user else "root",
+        )
+
+    def strategies(self) -> list[Any]:
+        """The strategies a session with no address can use, in rank order."""
+        from ..transport.strategies import ProviderFallback, QuicUDP, TailcatUDP, TCPPunch
+
+        return [TCPPunch(), TailcatUDP(), QuicUDP(), ProviderFallback(rank=6)]
+
+    def link_over(self, channel: Any) -> Any:
+        """Race the strategies over the bridge and return the link that wins."""
+        from ..transport.announce import printer
+        from ..transport.pipeline import LinkCache, Pipeline, network_fingerprint
+
+        target = self.target_over(channel)
+        return Pipeline(
+            self.strategies(),
+            target=target,
+            alias=self.alias,
+            cache=LinkCache(self.alias),
+            fingerprint=lambda: network_fingerprint(target.stun),
+            say=printer(self.announce),
+            floor=self.link_floor,
+        ).connect()
+
+    @property
+    def link_floor(self) -> Any:
+        """The slowest probe this account accepts, as a shell account's does."""
+        from ..transport.pipeline import MIB, LinkFloor
+
+        default = LinkFloor.default()
+        min_mib = self.config.option("min_mib_per_s")
+        max_rtt = self.config.option("max_rtt_ms")
+        return LinkFloor(
+            min_bps=float(min_mib) * MIB if isinstance(min_mib, (int, float)) else default.min_bps,
+            max_rtt_ms=float(max_rtt) if isinstance(max_rtt, (int, float)) else default.max_rtt_ms,
+        )
+
+    @property
+    def remote_python(self) -> str:
+        value = self.config.option("remote_python")
+        return str(value) if value else "python3"
+
+    def channel_over(self, bridge: Any, *, name: str) -> Channel:
+        """A worker over the chosen link, or the bridge when nothing was chosen.
+
+        Spec "Kaggle runtimes", An SSH link over the kernel: a network that cannot punch
+        keeps the bridge, which is what every Kaggle session used until now.
+        """
+        import shlex as _shlex
+
+        from ..protocol.worker import BOOTSTRAP
+        from ..runtime.channel import PersistentChannel
+
+        try:
+            link = self.link_over(bridge)
+        except Exception:
+            return bridge
+        self._links[name] = link
+        return PersistentChannel(
+            link.ssh_command(f"{self.remote_python} -u -c {_shlex.quote(BOOTSTRAP)}"),
+            name=name,
+        )
+
     def open_channel(self, runtime: Runtime) -> Channel:
         """Start a session from the cookie and open one worker in one cell of it.
 
@@ -1065,7 +1230,9 @@ class Kaggle(Provider):
             session=session,
         )
         self._kernels[runtime.name] = (session, kernel, channel, run, notebook, slug)
-        return channel
+        # Spec "Kaggle runtimes", An SSH link over the kernel: the bridge is the
+        # rendezvous, and the session rides a link when one is chosen over it.
+        return self.channel_over(channel, name=runtime.name)
 
     def stop(self, runtime: Runtime) -> None:
         """Close the bridge, delete the kernel, cancel the session run, delete the notebook.
@@ -1081,6 +1248,12 @@ class Kaggle(Provider):
         if held is None:
             return
         session, kernel, channel, run, notebook, slug = held
+        link = self._links.pop(runtime.name, None)
+        if link is not None:
+            try:
+                link.close()
+            except Exception:
+                pass
         try:
             channel.close()
         except OSError:

{letify-1.2.4 → letify-2.0.0}/letify/providers/local.py

@@ -143,6 +143,20 @@ class Local(Provider):
 
         return PersistentChannel([str(interpreter), "-u", "-c", BOOTSTRAP], name=runtime.name)
 
+    def transfer_channel(self, runtime: Runtime | None) -> Channel:
+        """Another worker subprocess for one transfer stream.
+
+        Spec "Several connections at once" is about kernel connections, and here there is
+        no network to make one over. A subprocess is the local equivalent, and having it
+        lets the parallel placement be exercised over the real framed path.
+        """
+        from ..protocol.worker import BOOTSTRAP
+        from ..runtime.channel import PersistentChannel
+
+        return PersistentChannel(
+            [str(sys.executable), "-u", "-c", BOOTSTRAP], name=f"{self.alias}-transfer"
+        )
+
     def device_channel(self, runtime: Runtime) -> Channel:
         """The session's call worker subprocess, which hosts the PyTorch device executor."""
         assert runtime.channel is not None

{letify-1.2.4 → letify-2.0.0}/letify/providers/shell.py

@@ -140,6 +140,28 @@ class Shell(Provider):
         value = self.config.option("reverse_ssh")
         return dict(value) if isinstance(value, Mapping) else None
 
+    @property
+    def connect_back(self) -> dict[str, Any] | None:
+        """Where the remote can reach this machine, from ``connect_back``.
+
+        Spec "Connect back". ``port`` is required; ``address`` is optional and the STUN
+        answer stands in for it when the port is forwarded here but the name is not known.
+        """
+        value = self.config.option("connect_back")
+        if not isinstance(value, Mapping) or not value.get("port"):
+            return None
+        return dict(value)
+
+    @property
+    def port_mapping(self) -> bool:
+        """Whether to ask the NAT for a forwarding, from ``port_mapping``.
+
+        Spec "Asking the NAT for a mapping": on by default, because a device that does not
+        speak one of the three protocols answers nothing and the attempt costs a second.
+        """
+        value = self.config.option("port_mapping")
+        return True if value is None else bool(value)
+
     @property
     def link_floor(self) -> LinkFloor:
         """The slowest probe this account accepts, from ``min_mib_per_s`` and ``max_rtt_ms``.
@@ -231,17 +253,22 @@ class Shell(Provider):
     def strategies(self) -> list[Strategy]:
         """The ranked strategy list. Reverse SSH joins only when the account sets it."""
         from ..transport.strategies import (
+            ConnectBack,
             DirectSSH,
             ProviderFallback,
+            QuicUDP,
             ReverseSSH,
             TailcatUDP,
             TCPPunch,
         )
 
-        chosen: list[Strategy] = [DirectSSH(), TCPPunch(), TailcatUDP()]
+        chosen: list[Strategy] = [DirectSSH()]
+        if self.connect_back or self.port_mapping:
+            chosen.append(ConnectBack(mapping=self.port_mapping))
+        chosen.extend((TCPPunch(), TailcatUDP(), QuicUDP()))
         if self.reverse_ssh:
             chosen.append(ReverseSSH())
-        chosen.append(ProviderFallback(rank=5 if self.reverse_ssh else 4))
+        chosen.append(ProviderFallback(rank=7 if self.reverse_ssh else 6))
         return chosen
 
     def fallback(self, runtime: Runtime | None = None) -> Callable[[], Link] | None:
@@ -269,6 +296,7 @@ class Shell(Provider):
             remote_python=self.remote_python,
             rendezvous=self.rendezvous(runtime),
             reverse_ssh=self.reverse_ssh,
+            connect_back=self.connect_back,
             fallback=self.fallback(runtime),
             stun=stun,
             tailcat=self.tailcat_binary,
@@ -463,6 +491,42 @@ class Shell(Provider):
             name=runtime.name,
         )
 
+    def transfer_command(self, runtime: Runtime | None) -> list[str]:
+        """An SSH command for one transfer connection, outside the multiplexed one.
+
+        Spec "Several connections at once": every SSH command letify builds carries
+        ``ControlMaster=auto``, which puts further sessions inside the first connection.
+        A transfer stream has to be its own kernel connection, so it turns that off. A
+        punched link redials for each, which is already one connection per session.
+        """
+        import shlex as _shlex
+
+        from ..protocol.worker import BOOTSTRAP
+
+        link = self.link(runtime)
+        command = link.ssh_command(
+            self.remote_command(f"{self.remote_python} -u -c {_shlex.quote(BOOTSTRAP)}")
+        )
+        kept: list[str] = []
+        index = 0
+        while index < len(command):
+            part = command[index]
+            if part == "-o" and index + 1 < len(command):
+                option = command[index + 1]
+                if option.startswith(("ControlMaster=", "ControlPath=", "ControlPersist=")):
+                    index += 2
+                    continue
+            kept.append(part)
+            index += 1
+        # After the program name, so they are options rather than the remote command.
+        return [kept[0], "-o", "ControlMaster=no", "-o", "ControlPath=none", *kept[1:]]
+
+    def transfer_channel(self, runtime: Runtime | None) -> Channel:
+        """One channel for one transfer stream, on a connection of its own."""
+        from ..runtime.channel import PersistentChannel
+
+        return PersistentChannel(self.transfer_command(runtime), name=f"{self.alias}-transfer")
+
     def device_channel(self, runtime: Runtime) -> Channel:
         """The session's call channel, which hosts the PyTorch device executor."""
         from ..errors import UnsupportedMode