ms-env 0.3.2__tar.gz → 0.3.3__tar.gz

{ms_env-0.3.2 → ms_env-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ms_env
-Version: 0.3.2
+Version: 0.3.3
 Summary: Environments for MolSuite apps: find or install external tools, on this machine or over ssh.
 Author: Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco
 License-Expression: MIT
@@ -125,7 +125,8 @@ as it is.
 `Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
 `ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
 logs in or the command fails at once, no credential is handled and nothing is left
-listening on the other side. A remote host is a POSIX machine: every command there runs through `sh`. This
+listening on the other side. Outside Windows the commands to one host share one connection
+(a socket in `~/.ssh`, closed a minute after the last command). A remote host is a POSIX machine: every command there runs through `sh`. This
 machine needs no `sh`: what ms_env does here it does in Python, Windows included.
 
 ## The environment of an app

{ms_env-0.3.2 → ms_env-0.3.3}/README.md

@@ -100,7 +100,8 @@ as it is.
 `Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
 `ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
 logs in or the command fails at once, no credential is handled and nothing is left
-listening on the other side. A remote host is a POSIX machine: every command there runs through `sh`. This
+listening on the other side. Outside Windows the commands to one host share one connection
+(a socket in `~/.ssh`, closed a minute after the last command). A remote host is a POSIX machine: every command there runs through `sh`. This
 machine needs no `sh`: what ms_env does here it does in Python, Windows included.
 
 ## The environment of an app

{ms_env-0.3.2 → ms_env-0.3.3}/src/ms_env/app.py

@@ -71,23 +71,29 @@ def _wheel(root: Path, tag: str, cache: Path) -> Path:
     return next(out.glob("*.whl"))
 
 
-def _constraints(everything: bool) -> str:
-    """`name==version` of what this environment holds: all of it, or what pip took from an index.
+def _constraints() -> tuple[str, str]:
+    """`name==version` of what this environment holds: all of it, and what pip took from an index.
 
     A local build is never pinned (no index has it), nor a name this environment has two
-    records of (which one runs is anyone's guess). What conda put here is pinned only when
-    `everything`: an index may lack that version, which is why the install has a second try.
+    records of (which one runs is anyone's guess). What conda put here is pinned only in the
+    first: an index may lack that version, which is why the install has a second try.
     """
-    seen: dict[str, set[str]] = {}
-    for dist in metadata.distributions():
+    seen: dict[str, set[tuple[str, bool]]] = {}
+    for dist in metadata.distributions():  # one pass: reading every record is most of what this costs
         name, installer = dist.metadata.get("Name"), (dist.read_text("INSTALLER") or "").strip()
         if not name:  # a broken leftover in site-packages
             continue
         local = "+" in dist.version or (installer != "conda" and dist.read_text("direct_url.json"))
-        pin = not local and (everything or installer in ("pip", "uv"))
-        seen.setdefault(re.sub(r"[-_.]+", "-", name).lower(), set()).add(dist.version if pin else "")
-    pins = (f"{name}=={next(iter(versions))}" for name, versions in seen.items() if len(versions) == 1)
-    return "\n".join(sorted(pin for pin in pins if not pin.endswith("=="))) + "\n"
+        seen.setdefault(re.sub(r"[-_.]+", "-", name).lower(), set()).add(
+            ("" if local else dist.version, installer in ("pip", "uv")))
+
+    def pins(everything: bool) -> str:
+        versions = {name: {version if everything or by_pip else "" for version, by_pip in records}
+                    for name, records in seen.items()}
+        return "\n".join(sorted(f"{name}=={next(iter(found))}" for name, found in versions.items()
+                                if len(found) == 1 and next(iter(found)))) + "\n"
+
+    return pins(True), pins(False)
 
 
 def app_recipe(
@@ -125,8 +131,9 @@ def app_recipe(
         wanted.append(f"{code}/{wheels[-1].name}{extras or ''}")
     cache.mkdir(parents=True, exist_ok=True)
     constraints, loose = cache / "constraints.txt", cache / "constraints-pip.txt"
-    constraints.write_text(_constraints(True), encoding="utf-8")
-    loose.write_text(_constraints(False), encoding="utf-8")
+    everything, by_pip = _constraints()
+    constraints.write_text(everything, encoding="utf-8")
+    loose.write_text(by_pip, encoding="utf-8")
 
     pip = (f"{prefix}/bin/python", "-m", "pip", "install", "--progress-bar", "off")
     # ponytail: one version the index lacks drops every pin of what conda installed here, not

{ms_env-0.3.2 → ms_env-0.3.3}/src/ms_env/host.py

@@ -2,9 +2,12 @@
 
 No credential is handled here. `ssh` runs with `BatchMode=yes`: the keys and the agent of the
 account log in or the command fails at once, and nothing is left listening on the other side.
+Where `ssh` can (not Windows' OpenSSH), the commands to one host share one connection, kept a
+minute after the last one: a check is then a few milliseconds instead of a new login.
 """
 from __future__ import annotations
 
+import os
 import platform as _platform
 import subprocess
 from pathlib import Path
@@ -23,6 +26,11 @@ def conda_platform(system: str, machine: str) -> str:
     return _PLATFORMS.get((system, machine), f"{system}-{machine}")
 
 
+# The shared connection's socket goes in ~/.ssh; ssh refuses to run when it cannot make it there.
+_SHARED = () if os.name == "nt" else (
+    "-o", "ControlMaster=auto", "-o", "ControlPersist=60", "-o", "ControlPath=~/.ssh/ms_env-%C")
+
+
 class HostError(RuntimeError):
     pass
 
@@ -31,8 +39,10 @@ class Host:
     """A machine, known by what running a command on it gives back: `address` as `ssh` knows it (its
     keys, users and jumps are in `~/.ssh/config`), or None for this one."""
 
-    def __init__(self, address: str | None = None, ssh=("ssh", "-o", "BatchMode=yes")):
+    def __init__(self, address: str | None = None, ssh=("ssh", "-o", "BatchMode=yes", *_SHARED)):
         self.address, self.ssh = address or None, tuple(ssh)
+        if self.address and _SHARED and _SHARED[-1] in self.ssh:
+            (Path.home() / ".ssh").mkdir(mode=0o700, exist_ok=True)
 
     def __repr__(self) -> str:
         return f"Host({self.address!r})"

{ms_env-0.3.2 → ms_env-0.3.3}/src/ms_env/recipes.py

@@ -74,7 +74,7 @@ from pathlib import Path, PurePath
 from typing import Any, Callable, Iterable
 
 from ms_env.steps import Step, StepFailed, find_environment_manager, run_step
-from ms_env.host import LOCAL, Host
+from ms_env.host import LOCAL, Host, HostError
 
 READY = "ready"
 MISSING = "missing"
@@ -386,7 +386,8 @@ def _base(owner: str) -> str:
 
 
 def _executable(info: tarfile.TarInfo) -> tarfile.TarInfo:
-    return info.replace(mode=0o755)  # what Windows packs carries no mode; a download has to run there
+    # not deep: a deep copy takes the TarFile it points to along, with all it packed so far
+    return info.replace(mode=0o755, deep=False)  # what Windows packs carries no mode; a download has to run there
 
 
 def _tar(paths: Iterable[str]) -> bytes:
@@ -397,8 +398,43 @@ def _tar(paths: Iterable[str]) -> bytes:
     return held.getvalue()
 
 
+# `_fetch`, run there by the host's own python3: the same check and unpacking, with no ms_env there yet.
+_FETCH_THERE = """import hashlib, os, shutil, sys, tempfile, urllib.request
+url, sha256, to = sys.argv[1:4]
+name = url.rsplit("/", 1)[-1]
+with tempfile.TemporaryDirectory() as held:
+    file, digest = os.path.join(held, name), hashlib.sha256()
+    with urllib.request.urlopen(url, timeout=60) as got, open(file, "wb") as out:
+        for chunk in iter(lambda: got.read(1 << 20), b""):
+            digest.update(chunk)
+            out.write(chunk)
+    if digest.hexdigest() != sha256:
+        sys.exit(f"{url} is not the file the recipe names: sha256 {digest.hexdigest()}, not {sha256}")
+    if any(name.endswith(ext) for _, extensions, _ in shutil.get_unpack_formats() for ext in extensions):
+        try:
+            shutil.unpack_archive(file, to, **({} if name.endswith(".zip") else {"filter": "data"}))
+        except TypeError:  # a python there older than the filter
+            shutil.unpack_archive(file, to)
+    else:
+        os.makedirs(os.path.dirname(to) or ".", exist_ok=True)
+        shutil.move(file, to)
+        os.chmod(to, 0o755)
+"""
+
+
+def _fetch_there(step: RecipeStep, host: Host, emit: Callable[[str], None]) -> None:
+    """A download for a remote host: made there when the host reaches the URL, else made here and sent."""
+    emit(f"downloading {step.url} on {host.address}")
+    try:
+        host.run(shlex.join(["python3", "-", step.url, step.sha256, step.to]), data=_FETCH_THERE.encode(),
+                 timeout=step.timeout)
+    except (HostError, subprocess.TimeoutExpired) as exc:
+        emit(f"{host.address} could not download it ({str(exc).splitlines()[-1][:200]}): sending it from here")
+        _fetch_and_send(step, host, emit)
+
+
 def _fetch_and_send(step: RecipeStep, host: Host, emit: Callable[[str], None]) -> None:
-    """A download for a remote host: made and checked here, then sent, so the host needs no curl."""
+    """A download for a host that has no python3 or no way out: made and checked here, then sent."""
     with tempfile.TemporaryDirectory() as held:
         _fetch(step.url, step.sha256, str(Path(held, posixpath.basename(step.to))), emit)
         parent = shlex.quote(posixpath.dirname(step.to))
@@ -412,7 +448,7 @@ def _step(step: RecipeStep, label: str, host: Host) -> Step:
         return Step(label + step.name, run=step.local, timeout=step.timeout)
     if step.url:
         run = partial(_fetch, step.url, step.sha256, step.to) if host.address is None else \
-            partial(_fetch_and_send, step, host)
+            partial(_fetch_there, step, host)
         return Step(label + step.name, run=run, timeout=step.timeout)
     if not step.send:
         return Step(label + step.name, argv=_argv_on(step.argv, host), timeout=step.timeout)
@@ -635,10 +671,15 @@ class Install:
     key: str
     future: Future = field(default_factory=Future)
     lines: list[str] = field(default_factory=list)  # live log, appended from the install thread
+    # Steps done of all it takes: the check, and then each step of an install when one is needed.
+    done: int = 0
+    total: int = 1
+    step: str = ""  # the name of the step it is running, while it runs one
 
 
 _INSTALLS: dict[str, Install] = {}
 _LOCK = threading.Lock()
+_CHANGING: dict[str, threading.Lock] = {}  # by host: the checks run side by side, the installs one at a time
 
 
 def ensure(recipe: Recipe, host: Host = LOCAL, busy: Callable[[], str] | None = None) -> Install:
@@ -682,8 +723,17 @@ def _run_install(install: Install, recipe: Recipe, host: Host, busy: Callable[[]
             why = busy() if busy and owner.startswith("{") else ""  # a stamp: an install of ours that ended
             if why:
                 raise StepFailed(f"{recipe.name} on {where} is not as this recipe asks, and is left as it is: {why}")
-            for step in recipe_steps(recipe, host, refresh=bool(recipe.base) and _base(owner) == recipe.base):
-                run_step(step, install.lines.append)
+            steps = recipe_steps(recipe, host, refresh=bool(recipe.base) and _base(owner) == recipe.base)
+            install.done, install.total = 1, 1 + len(steps)
+            with _LOCK:
+                changing = _CHANGING.setdefault(host.address or "", threading.Lock())
+            # ponytail: one install at a time on a host (they share its micromamba and its package cache)
+            with changing:
+                for step in steps:
+                    install.step = step.name
+                    run_step(step, install.lines.append)
+                    install.done += 1
+                install.step = ""
             state = status(recipe, host)
             if state != READY:
                 raise StepFailed(f"{recipe.name}: every step ran but the tool is still {state} on {where}.")
@@ -691,6 +741,7 @@ def _run_install(install: Install, recipe: Recipe, host: Host, busy: Callable[[]
         install.lines.append(str(exc))
         install.future.set_exception(exc)
     else:
+        install.done = install.total
         install.future.set_result(None)
 
 

{ms_env-0.3.2 → ms_env-0.3.3}/src/ms_env.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ms_env
-Version: 0.3.2
+Version: 0.3.3
 Summary: Environments for MolSuite apps: find or install external tools, on this machine or over ssh.
 Author: Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco
 License-Expression: MIT
@@ -125,7 +125,8 @@ as it is.
 `Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
 `ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
 logs in or the command fails at once, no credential is handled and nothing is left
-listening on the other side. A remote host is a POSIX machine: every command there runs through `sh`. This
+listening on the other side. Outside Windows the commands to one host share one connection
+(a socket in `~/.ssh`, closed a minute after the last command). A remote host is a POSIX machine: every command there runs through `sh`. This
 machine needs no `sh`: what ms_env does here it does in Python, Windows included.
 
 ## The environment of an app

ms_env-0.3.3/src/ms_env.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.3.3",
+  "distance": 0,
+  "node": "ga04372b97f9cb0bb6669d7e33a69accd2ad77471",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-10-29"
+}

{ms_env-0.3.2 → ms_env-0.3.3}/tests/test_ms_env.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 
 import subprocess
 import dataclasses
+import hashlib
+import os
 import json
 import sys
 from pathlib import Path
@@ -57,7 +59,7 @@ def test_spec_binds_per_host(tmp_path):
     assert [step.name.removeprefix("mario@login.hpc: ") for step in steps] == [
         "prepare /opt/mine", "create the faketool environment", "pip install into faketool", "record the faketool install",
     ]
-    assert steps[1].argv[:4] == ["ssh", "-o", "BatchMode=yes", "mario@login.hpc"]
+    assert steps[1].argv[:3] + [steps[1].argv[-2]] == ["ssh", "-o", "BatchMode=yes", "mario@login.hpc"]
     assert "/scratch/tools/micromamba" in steps[1].argv[-1]  # bootstrap lands under the remote home
     assert steps[2].argv[-1].endswith("'extra pkg'")  # one quoted remote command line
     assert remote.stamp == local.stamp  # what the TOML asks for, wherever it is applied
@@ -130,6 +132,32 @@ def test_install_needs_no_project(tmp_path, monkeypatch):
     assert "boom detail" in broken.lines
 
 
+def test_installs_on_one_host_take_turns_and_count_their_steps(tmp_path, monkeypatch):
+    import time
+
+    monkeypatch.setenv("PATH", "/usr/bin:/bin")
+    recipes = []
+    for name in ("a", "b"):
+        (tmp_path / name).mkdir()
+        recipes.append(dataclasses.replace(_plain_spec(tmp_path / name, slow=True), name=f"faketool-{name}"))
+    start = time.monotonic()
+    installs = [tools.ensure(recipe) for recipe in recipes]
+    running = set()
+    while not all(install.future.done() for install in installs):
+        running.update(install.step for install in installs)
+        time.sleep(0.05)
+    for install in installs:
+        install.future.result(timeout=30)
+    assert "wait" in running, running
+    assert all(install.step == "" for install in installs)  # named while it runs, not after
+    assert time.monotonic() - start >= 2  # one `sleep 1` after the other: they share the host's micromamba
+    # the check, "prepare", the two steps of the recipe, the stamp
+    assert [(install.done, install.total) for install in installs] == [(5, 5), (5, 5)]
+    again = tools.ensure(recipes[0])
+    again.future.result(timeout=30)
+    assert (again.done, again.total) == (1, 1)  # only the check: nothing to install
+
+
 def test_ms_env_updates_and_removes_what_it_installed(tmp_path, monkeypatch):
     monkeypatch.setenv("PATH", "/usr/bin:/bin")
     recipe = _plain_spec(tmp_path)
@@ -186,7 +214,8 @@ def test_the_environment_of_the_user_is_verified_and_never_changed(tmp_path, mon
 
 def test_a_host_runs_commands_and_says_why_one_failed():
     assert Host().run("cat; echo there", b"here ") == b"here there\n"
-    assert Host("login.hpc").argv("ls") == ["ssh", "-o", "BatchMode=yes", "login.hpc", "ls"]
+    argv = Host("login.hpc").argv("ls")
+    assert argv[:3] + argv[-2:] == ["ssh", "-o", "BatchMode=yes", "login.hpc", "ls"]
     with pytest.raises(tools.HostError, match="this machine: no such thing"):
         Host().run("echo no such thing >&2; exit 3")
 
@@ -294,7 +323,7 @@ def test_a_download_is_picked_per_platform_checked_and_installed(tmp_path, monke
         tools.ToolSpec.load(tmp_path / "bad.toml")
 
 
-def test_an_archive_is_unpacked_and_a_remote_host_gets_the_download_sent(tmp_path, monkeypatch):
+def test_an_archive_is_unpacked_and_a_remote_host_with_no_way_out_gets_the_download_sent(tmp_path, monkeypatch):
     import tarfile
     import zipfile
 
@@ -307,7 +336,13 @@ def test_an_archive_is_unpacked_and_a_remote_host_gets_the_download_sent(tmp_pat
     assert Path(recipe.prefix, "index.html").read_text() == "<html/>"
 
     sent: list[tuple[str, bytes]] = []
-    monkeypatch.setattr(Host, "run", lambda self, command, data=b"", timeout=0: sent.append((command, data)) or b"")
+
+    def run(self, command, data=b"", timeout=0):  # a host with no way out: what it cannot download is sent
+        if command.startswith("python3"):
+            raise tools.HostError("no route to host")
+        return sent.append((command, data)) or b""
+
+    monkeypatch.setattr(Host, "run", run)
     far = _download_spec(tmp_path, {"": _fake_exe(tmp_path / "release/faketool")}).recipe("/far/tools")
     run_step(tools.recipe_steps(far, Host("login.hpc"))[1])
     (command, data), = sent
@@ -343,3 +378,45 @@ def test_a_new_tag_on_the_same_code_builds_a_new_wheel(tmp_path):
     git("tag", "v0.2.0")  # a release of the same code: its wheel has another version
     assert before.startswith("v0.1.0-1-g") and app._described(tmp_path).startswith("v0.2.0-0-g")
     assert app._described(tmp_path / "not-a-checkout") == ""
+
+
+def test_a_folder_of_many_files_is_sent_as_a_tar_of_its_size(tmp_path):
+    import resource
+    import tarfile
+    from io import BytesIO
+
+    from ms_env.recipes import _tar
+
+    for i in range(300):  # each file once took a deep copy of all packed before it: GBs by 100
+        (tmp_path / "pack" / f"sub{i % 10}").mkdir(parents=True, exist_ok=True)
+        (tmp_path / "pack" / f"sub{i % 10}" / f"f{i}.js").write_bytes(b"x" * 20_000)
+    before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+    data = _tar([str(tmp_path / "pack")])
+    assert resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - before < 200 * 1024  # kB
+    with tarfile.open(fileobj=BytesIO(data)) as tar:
+        members = tar.getmembers()
+    assert len([m for m in members if m.isfile()]) == 300 and all(m.mode == 0o755 for m in members)
+
+
+def test_a_host_downloads_for_itself_and_is_sent_the_file_only_when_it_cannot(tmp_path):
+    from ms_env.recipes import RecipeStep, _step
+
+    payload = tmp_path / "tool.bin"
+    payload.write_bytes(b"#!/bin/sh\necho hi\n")
+    digest = hashlib.sha256(payload.read_bytes()).hexdigest()
+
+    class Far(Host):  # a "remote" host that is this machine; `blocked` is a host with no way out
+        blocked = False
+
+        def argv(self, command):
+            if self.blocked and command.startswith("python3"):
+                return ["sh", "-c", "echo 'no route to host' >&2; exit 1"]
+            return ["sh", "-c", command]
+
+    for blocked in (False, True):
+        host, said = Far("far"), []
+        host.blocked = blocked
+        to = tmp_path / f"there-{blocked}" / "tool"
+        _step(RecipeStep("get", (), 60.0, url=payload.as_uri(), sha256=digest, to=str(to)), "", host).run(said.append)
+        assert to.read_bytes() == payload.read_bytes() and os.access(to, os.X_OK)
+        assert any("sending it from here" in line for line in said) == blocked

ms_env-0.3.2/src/ms_env.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.3.2",
-  "distance": 0,
-  "node": "g3f644bbfcaf555dc7cedf971907597ad44ef3f75",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-10-28"
-}