letify 2.4.0__tar.gz → 2.4.2__tar.gz

{letify-2.4.0 → letify-2.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: letify
-Version: 2.4.0
+Version: 2.4.2
 Summary: Declarations that become infrastructure.
 Project-URL: Homepage, https://github.com/DarkPyonix/letify
 Project-URL: Documentation, https://github.com/DarkPyonix/letify/tree/main/docs

{letify-2.4.0 → letify-2.4.2}/letify/__init__.py

@@ -62,7 +62,7 @@ remote: Final = Host.remote
 # The enum class stays importable from letify.declare.instance for letify's own use.
 del Host
 
-__version__ = "2.4.0"
+__version__ = "2.4.2"
 
 
 def fetch(tensor):  # type: ignore[no-untyped-def]

{letify-2.4.0 → letify-2.4.2}/letify/cli.py

@@ -40,6 +40,16 @@ def build_parser() -> argparse.ArgumentParser:
     sessions_parser.add_argument(
         "--json", action="store_true", help="print the records unformatted"
     )
+    stop = sub.add_parser("stop", help="stop what a dead process left behind")
+    stop.add_argument(
+        "--orphans",
+        action="store_true",
+        help="delete the sessions whose creating process is gone",
+    )
+    stop.add_argument("--alias", help="only this account")
+    stop.add_argument("-y", "--yes", action="store_true", help="do not ask")
+    stop.add_argument("--json", action="store_true", help="print the records unformatted")
+
     sub.add_parser("stubs", help="write the provider types an editor completes")
 
     usage = sub.add_parser("usage", help="show what each account has left")
@@ -523,6 +533,25 @@ def _say(kind: str, message: str) -> None:
     print(f"{render.mark(kind, _out())} {message}")
 
 
+def _detail(row: dict) -> str:
+    """The reason a stop failed, when there is one to add."""
+    return f": {row['error']}" if row.get("error") else ""
+
+
+def _confirmed(question: str) -> bool:
+    """Ask before deleting. Anything but yes is no, and no terminal is no."""
+    import sys as _sys
+
+    if not _sys.stdin.isatty():
+        print(f"{question} not a terminal, so nothing is assumed. Pass -y to delete")
+        return False
+    try:
+        answer = input(f"{question} [y/N] ").strip().lower()
+    except EOFError:
+        return False
+    return answer in ("y", "yes")
+
+
 def _json(value: object) -> int:
     print(json.dumps(value, indent=2))
     return 0
@@ -679,6 +708,39 @@ def _dispatch(args: argparse.Namespace) -> int:
             print(f"{row['alias']}  {detail}")
         return 0
 
+    if args.command == "stop":
+        if not args.orphans:
+            # A bare `stop` reads as ending everything, so it ends nothing.
+            print("nothing was stopped. Pass --orphans to delete what a dead process left")
+            return 0
+        rows = let.orphans(args.alias)
+        found = [(row["alias"], ref) for row in rows for ref in row.get("orphans", ())]
+        unavailable = [row for row in rows if row.get("unavailable")]
+        if args.json and not found:
+            return _json(rows)
+        for row in unavailable:
+            print(f"{row['alias']}  {row['unavailable']}")
+        if not found:
+            print("no orphaned sessions")
+            return 0
+        print(f"{len(found)} orphaned session(s) to delete:")
+        for alias, ref in found:
+            print(f"  {alias}  {ref}")
+        if not args.yes and not _confirmed("Delete them?"):
+            print("nothing was stopped")
+            return 0
+        done = let.stop_orphans(args.alias)
+        if args.json:
+            return _json(done)
+        failed = 0
+        for row in done:
+            if row.get("unavailable"):
+                continue
+            mark = "stopped" if row.get("stopped") else f"failed{_detail(row)}"
+            failed += 0 if row.get("stopped") else 1
+            print(f"{row['alias']}  {row['ref']}  {mark}")
+        return 1 if failed else 0
+
     if args.command == "status":
         status = let.status()
         if args.json:

{letify-2.4.0 → letify-2.4.2}/letify/install.py

@@ -37,6 +37,10 @@ from .transport.setup import TAILCAT_VERSION
 
 #: Seconds to wait for a release server.
 DOWNLOAD_TIMEOUT = 60.0
+#: Seconds to wait for the connection alone. A host a network blocks has to fail in
+#: seconds, not after the read timeout, and twice that when a release redirects.
+#: Spec "Automatic install".
+CONNECT_TIMEOUT = 10.0
 
 #: Bytes read from a release server at a time, so progress can be reported.
 DOWNLOAD_CHUNK = 64 << 10
@@ -352,8 +356,35 @@ def _isatty(stream) -> bool:
         return False
 
 
+def _reachable(url: str) -> None:
+    """Refuse a download whose host does not answer, before the request is made.
+
+    Spec "Automatic install": the error names the host so a blocked network is told what
+    to open or to configure around, rather than being told the tool is broken.
+    """
+    import socket
+    import urllib.parse
+
+    parsed = urllib.parse.urlsplit(url)
+    host = parsed.hostname
+    if not host:
+        return
+    port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    try:
+        socket.create_connection((host, port), timeout=CONNECT_TIMEOUT).close()
+    except OSError as exc:
+        raise InstallError(
+            f"could not reach {host}:{port} within {CONNECT_TIMEOUT:.0f} s ({exc}), so "
+            f"{url} was not downloaded. The host, not letify, is what is unreachable: a "
+            f"network that allows github.com can still block the host its releases "
+            f"redirect to. Fetch the archive another way and name the binary with the "
+            f"account's tailcat_binary or eci_binary"
+        ) from None
+
+
 def _download(url: str, *, label: str | None = None, stream=None) -> bytes:
     """Fetch ``url``; with ``label``, report progress on ``stream`` (standard error)."""
+    _reachable(url)
     try:
         with urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT) as response:
             if label is None:

{letify-2.4.0 → letify-2.4.2}/letify/launcher.py

@@ -499,6 +499,40 @@ class Launcher:
                 rows.append({"alias": alias, "sessions": [], "unavailable": str(exc)})
         return rows
 
+    def orphans(self, alias: str | None = None) -> list[dict[str, Any]]:
+        """What a dead process left behind, per account.
+
+        Spec "Kaggle", stopping what a dead process left behind.
+        """
+        rows: list[dict[str, Any]] = []
+        for name in self.config.order:
+            if alias is not None and name != alias:
+                continue
+            try:
+                provider = self.provider(name)
+                rows.append({"alias": name, "kind": provider.kind, "orphans": provider.orphans()})
+            except LetifyError as exc:
+                rows.append({"alias": name, "orphans": [], "unavailable": str(exc)})
+        return rows
+
+    def stop_orphans(self, alias: str | None = None) -> list[dict[str, Any]]:
+        """Delete what ``orphans`` reported, one at a time, reporting each outcome."""
+        done: list[dict[str, Any]] = []
+        for row in self.orphans(alias):
+            if row.get("unavailable"):
+                done.append(row)
+                continue
+            provider = self.provider(row["alias"])
+            for ref in row["orphans"]:
+                try:
+                    gone = provider.stop_orphan(ref)
+                    done.append({"alias": row["alias"], "ref": ref, "stopped": gone})
+                except LetifyError as exc:
+                    done.append(
+                        {"alias": row["alias"], "ref": ref, "stopped": False, "error": str(exc)}
+                    )
+        return done
+
     def status(self) -> dict[str, Any]:
         """What is running right now, and what it is costing.
 

{letify-2.4.0 → letify-2.4.2}/letify/protocol/worker.py

@@ -953,9 +953,12 @@ def _data_register(data):
             # A call still waiting for bytes needs the patch to answer from the manifest,
             # and a call that placed a link needs it for copy on write. Spec "Copy on write".
             _data_install_patch()
-        if state["pending"]:
-            # Only a call still waiting for bytes needs the manifest on disk and the
-            # wrappers. Inside the lock: the data thread moves a path out of pending and
+        if state["pending"] or state["linked"]:
+            # A call still waiting for bytes needs the manifest on disk for the listings,
+            # and a call that placed a link needs it so a process the body starts can
+            # convert that link before writing through it: the patch a fresh process
+            # installs reads the layout from the manifest and installs nothing without it.
+            # Spec "Copy on write". Inside the lock: the data thread moves a path out of pending and
             # into placed as each blob completes, and the manifest is written from both maps.
             _data_write_manifest(state)
             _data_observe(state)
@@ -1416,7 +1419,7 @@ def _data_write_manifest(state):
     """Write the call's manifest beside its files, for a process started fresh.
 
     Spec "What the body sees before a file arrives": a spawned interpreter installs the
-    same patch from ``letify_pending.pth`` and reads the layout from here, because it has
+    same patch from ``sitecustomize.py`` and reads the layout from here, because it has
     no channel of its own to ask on.
     """
     import json
@@ -1435,10 +1438,11 @@ def _data_write_manifest(state):
     directory = os.path.join(os.path.dirname(state["blobs"]), "pending")
     try:
         os.makedirs(directory, exist_ok=True)
-        with open(os.path.join(directory, "letify_pending.py"), "w") as handle:
+        # sitecustomize, not a .pth: site.py runs a .pth only in a directory it adds
+        # through addsitedir, and a PYTHONPATH entry is not one, so a .pth there never
+        # runs. Spec "What the body sees before a file arrives".
+        with open(os.path.join(directory, "sitecustomize.py"), "w") as handle:
             handle.write(_PENDING_MODULE)
-        with open(os.path.join(directory, "letify_pending.pth"), "w") as handle:
-            handle.write("import letify_pending\n")
     except OSError:
         return
     existing = os.environ.get("PYTHONPATH", "")
@@ -1457,6 +1461,8 @@ import builtins
 import io
 import json
 import os
+import shutil
+import sys
 import time
 
 
@@ -1474,13 +1480,25 @@ def _install():
     real_open, real_io_open = builtins.open, io.open
     real_listdir, real_scandir = os.listdir, os.scandir
     real_stat = os.stat
+    real_os_open = os.open
+    write_flags = os.O_WRONLY | os.O_RDWR | os.O_APPEND | os.O_TRUNC
+
+    # Whether the file exists, asked through the real stat. os.path.exists goes through
+    # os.stat, and os.stat is what this patch replaces, so asking it here would call the
+    # patch again and recurse without bottom.
+    def there(text):
+        try:
+            real_stat(text)
+        except OSError:
+            return False
+        return True
 
     def missing(path):
         try:
             text = os.path.abspath(os.fspath(path))
         except TypeError:
             return None
-        if not text.startswith(call_dir) or os.path.exists(text):
+        if not text.startswith(call_dir) or there(text):
             return None
         return text if text in files else None
 
@@ -1489,19 +1507,64 @@ def _install():
         if text is None:
             return
         deadline = time.monotonic() + wait
-        while not os.path.exists(text):
+        while not there(text):
             if time.monotonic() >= deadline:
                 raise RuntimeError("letify: %s did not arrive within %.0f s" % (text, wait))
             time.sleep(0.02)
 
+    # A placed file is a hard link to the cache blob, so a write through it writes into
+    # the cache. The worker converts the link first and this has to as well; it cannot read
+    # the worker's placed map, so a link count above one under the call directory is what
+    # it goes on. Spec "What the body sees before a file arrives".
+    def convert(path):
+        try:
+            text = os.path.abspath(os.fspath(path))
+        except TypeError:
+            return
+        if not text.startswith(call_dir):
+            return
+        try:
+            if real_stat(text).st_nlink < 2:
+                return
+        except OSError:
+            return
+        partial = '%s.letify-placing.%d' % (text, os.getpid())
+        try:
+            shutil.copyfile(text, partial)
+            os.replace(partial, text)
+        except OSError:
+            try:
+                os.remove(partial)
+            except OSError:
+                pass
+
+    def writes(mode):
+        return any(letter in mode for letter in ('w', 'a', 'x', '+'))
+
+    def ready(path, writing):
+        wait_for(path)
+        if writing:
+            convert(path)
+
     def opened(file, *args, **kwargs):
-        wait_for(file)
+        mode = kwargs['mode'] if 'mode' in kwargs else (args[0] if args else 'r')
+        ready(file, writes(mode) if isinstance(mode, str) else False)
         return real_open(file, *args, **kwargs)
 
     def io_opened(file, *args, **kwargs):
-        wait_for(file)
+        mode = kwargs['mode'] if 'mode' in kwargs else (args[0] if args else 'r')
+        ready(file, writes(mode) if isinstance(mode, str) else False)
         return real_io_open(file, *args, **kwargs)
 
+    def os_opened(path, *args, **kwargs):
+        flags = kwargs['flags'] if 'flags' in kwargs else (args[0] if args else 0)
+        try:
+            writing = bool(flags & write_flags)
+        except TypeError:
+            writing = False
+        ready(path, writing)
+        return real_os_open(path, *args, **kwargs)
+
     def names(path):
         try:
             text = os.path.abspath(os.fspath(path))
@@ -1534,12 +1597,37 @@ def _install():
 
     builtins.open = opened
     io.open = io_opened
+    os.open = os_opened
     os.listdir = listed
     os.stat = stated
     os.scandir = real_scandir
 
 
+# letify's directory goes first on PYTHONPATH, so it shadows a sitecustomize the user
+# installed. That one still has to run.
+def _chain():
+    import importlib.util
+
+    here = os.path.dirname(os.path.abspath(__file__))
+    for entry in sys.path:
+        if not entry:
+            continue
+        try:
+            if os.path.abspath(entry) == here:
+                continue
+            candidate = os.path.join(entry, 'sitecustomize.py')
+            if not os.path.isfile(candidate):
+                continue
+            spec = importlib.util.spec_from_file_location('letify_other_sitecustomize', candidate)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+        except Exception:
+            pass
+        return
+
+
 _install()
+_chain()
 """
 
 
@@ -2126,12 +2214,18 @@ def _run(stream, request, settle):
     request = None
     if settle:
         _settle()
-    if name == "call":
-        # Spec "Worker output": the body's output is framed before its result goes out.
+    if name in _DRAINS:
+        # Spec "Worker output": what the program wrote is framed before its result goes
+        # out. Every op that runs a program drains; a stat or a lease renewal does not,
+        # because they write nothing and are on the hot path.
         _drain_output()
     _reply(stream, outcome)
 
 
+#: Ops that run a program, so output of theirs may still be unframed when they answer.
+#: Spec "Worker output".
+_DRAINS = frozenset(("call", "exec"))
+
 _DATA_PREFIX = b"LETIFY-DATA "
 
 

{letify-2.4.0 → letify-2.4.2}/letify/providers/base.py

@@ -374,6 +374,18 @@ class Provider(abc.ABC):
         """
         return []
 
+    def orphans(self) -> list[str]:
+        """What a dead process left behind on this account, for a provider that can tell.
+
+        Spec "Kaggle", stopping what a dead process left behind. Empty for a provider that
+        cannot, so `letify stop --orphans` has nothing to delete there.
+        """
+        return []
+
+    def stop_orphan(self, ref: str) -> bool:
+        """Delete one thing `orphans` reported, and report whether it is gone."""
+        return False
+
     def expected_cards(self, instance: Any) -> tuple[str, int] | None:
         """The card name fragment and count a session for ``instance`` must have, or None.