ms-env 0.3.2__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. {ms_env-0.3.2 → ms_env-0.4.0}/PKG-INFO +3 -2
  2. {ms_env-0.3.2 → ms_env-0.4.0}/README.md +2 -1
  3. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/__init__.py +4 -0
  4. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/app.py +19 -12
  5. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/host.py +11 -1
  6. ms_env-0.4.0/src/ms_env/locking.py +149 -0
  7. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/recipes.py +71 -17
  8. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/PKG-INFO +3 -2
  9. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/SOURCES.txt +1 -0
  10. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/scm_file_list.json +1 -0
  11. ms_env-0.4.0/src/ms_env.egg-info/scm_version.json +8 -0
  12. {ms_env-0.3.2 → ms_env-0.4.0}/tests/test_ms_env.py +121 -4
  13. ms_env-0.3.2/src/ms_env.egg-info/scm_version.json +0 -8
  14. {ms_env-0.3.2 → ms_env-0.4.0}/.github/workflows/ci.yml +0 -0
  15. {ms_env-0.3.2 → ms_env-0.4.0}/.github/workflows/docs.yml +0 -0
  16. {ms_env-0.3.2 → ms_env-0.4.0}/.github/workflows/publish.yml +0 -0
  17. {ms_env-0.3.2 → ms_env-0.4.0}/.gitignore +0 -0
  18. {ms_env-0.3.2 → ms_env-0.4.0}/LICENSE +0 -0
  19. {ms_env-0.3.2 → ms_env-0.4.0}/docs/api_reference.md +0 -0
  20. {ms_env-0.3.2 → ms_env-0.4.0}/docs/guide.md +0 -0
  21. {ms_env-0.3.2 → ms_env-0.4.0}/docs/index.md +0 -0
  22. {ms_env-0.3.2 → ms_env-0.4.0}/pyproject.toml +0 -0
  23. {ms_env-0.3.2 → ms_env-0.4.0}/setup.cfg +0 -0
  24. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/steps.py +0 -0
  25. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/dependency_links.txt +0 -0
  26. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/requires.txt +0 -0
  27. {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/top_level.txt +0 -0
  28. {ms_env-0.3.2 → ms_env-0.4.0}/zensical.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ms_env
3
- Version: 0.3.2
3
+ Version: 0.4.0
4
4
  Summary: Environments for MolSuite apps: find or install external tools, on this machine or over ssh.
5
5
  Author: Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco
6
6
  License-Expression: MIT
@@ -125,7 +125,8 @@ as it is.
125
125
  `Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
126
126
  `ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
127
127
  logs in or the command fails at once, no credential is handled and nothing is left
128
- listening on the other side. A remote host is a POSIX machine: every command there runs through `sh`. This
128
+ listening on the other side. Outside Windows the commands to one host share one connection
129
+ (a socket in `~/.ssh`, closed a minute after the last command). A remote host is a POSIX machine: every command there runs through `sh`. This
129
130
  machine needs no `sh`: what ms_env does here it does in Python, Windows included.
130
131
 
131
132
  ## The environment of an app
@@ -100,7 +100,8 @@ as it is.
100
100
  `Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
101
101
  `ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
102
102
  logs in or the command fails at once, no credential is handled and nothing is left
103
- listening on the other side. A remote host is a POSIX machine: every command there runs through `sh`. This
103
+ listening on the other side. Outside Windows the commands to one host share one connection
104
+ (a socket in `~/.ssh`, closed a minute after the last command). A remote host is a POSIX machine: every command there runs through `sh`. This
104
105
  machine needs no `sh`: what ms_env does here it does in Python, Windows included.
105
106
 
106
107
  ## The environment of an app
@@ -22,6 +22,7 @@ from ms_env.recipes import (
22
22
  )
23
23
  from ms_env.steps import Step, StepFailed, find_environment_manager, run_step, run_steps
24
24
  from ms_env.host import LOCAL, Host, HostError, conda_platform
25
+ from ms_env.locking import Locked, prefix_lock, unlock
25
26
 
26
27
  __all__ = [
27
28
  "Install",
@@ -48,4 +49,7 @@ __all__ = [
48
49
  "run_steps",
49
50
  "status",
50
51
  "verify",
52
+ "prefix_lock",
53
+ "unlock",
54
+ "Locked",
51
55
  ]
@@ -71,23 +71,29 @@ def _wheel(root: Path, tag: str, cache: Path) -> Path:
71
71
  return next(out.glob("*.whl"))
72
72
 
73
73
 
74
- def _constraints(everything: bool) -> str:
75
- """`name==version` of what this environment holds: all of it, or what pip took from an index.
74
+ def _constraints() -> tuple[str, str]:
75
+ """`name==version` of what this environment holds: all of it, and what pip took from an index.
76
76
 
77
77
  A local build is never pinned (no index has it), nor a name this environment has two
78
- records of (which one runs is anyone's guess). What conda put here is pinned only when
79
- `everything`: an index may lack that version, which is why the install has a second try.
78
+ records of (which one runs is anyone's guess). What conda put here is pinned only in the
79
+ first: an index may lack that version, which is why the install has a second try.
80
80
  """
81
- seen: dict[str, set[str]] = {}
82
- for dist in metadata.distributions():
81
+ seen: dict[str, set[tuple[str, bool]]] = {}
82
+ for dist in metadata.distributions(): # one pass: reading every record is most of what this costs
83
83
  name, installer = dist.metadata.get("Name"), (dist.read_text("INSTALLER") or "").strip()
84
84
  if not name: # a broken leftover in site-packages
85
85
  continue
86
86
  local = "+" in dist.version or (installer != "conda" and dist.read_text("direct_url.json"))
87
- pin = not local and (everything or installer in ("pip", "uv"))
88
- seen.setdefault(re.sub(r"[-_.]+", "-", name).lower(), set()).add(dist.version if pin else "")
89
- pins = (f"{name}=={next(iter(versions))}" for name, versions in seen.items() if len(versions) == 1)
90
- return "\n".join(sorted(pin for pin in pins if not pin.endswith("=="))) + "\n"
87
+ seen.setdefault(re.sub(r"[-_.]+", "-", name).lower(), set()).add(
88
+ ("" if local else dist.version, installer in ("pip", "uv")))
89
+
90
+ def pins(everything: bool) -> str:
91
+ versions = {name: {version if everything or by_pip else "" for version, by_pip in records}
92
+ for name, records in seen.items()}
93
+ return "\n".join(sorted(f"{name}=={next(iter(found))}" for name, found in versions.items()
94
+ if len(found) == 1 and next(iter(found)))) + "\n"
95
+
96
+ return pins(True), pins(False)
91
97
 
92
98
 
93
99
  def app_recipe(
@@ -125,8 +131,9 @@ def app_recipe(
125
131
  wanted.append(f"{code}/{wheels[-1].name}{extras or ''}")
126
132
  cache.mkdir(parents=True, exist_ok=True)
127
133
  constraints, loose = cache / "constraints.txt", cache / "constraints-pip.txt"
128
- constraints.write_text(_constraints(True), encoding="utf-8")
129
- loose.write_text(_constraints(False), encoding="utf-8")
134
+ everything, by_pip = _constraints()
135
+ constraints.write_text(everything, encoding="utf-8")
136
+ loose.write_text(by_pip, encoding="utf-8")
130
137
 
131
138
  pip = (f"{prefix}/bin/python", "-m", "pip", "install", "--progress-bar", "off")
132
139
  # ponytail: one version the index lacks drops every pin of what conda installed here, not
@@ -2,9 +2,12 @@
2
2
 
3
3
  No credential is handled here. `ssh` runs with `BatchMode=yes`: the keys and the agent of the
4
4
  account log in or the command fails at once, and nothing is left listening on the other side.
5
+ Where `ssh` can (not Windows' OpenSSH), the commands to one host share one connection, kept a
6
+ minute after the last one: a check is then a few milliseconds instead of a new login.
5
7
  """
6
8
  from __future__ import annotations
7
9
 
10
+ import os
8
11
  import platform as _platform
9
12
  import subprocess
10
13
  from pathlib import Path
@@ -23,6 +26,11 @@ def conda_platform(system: str, machine: str) -> str:
23
26
  return _PLATFORMS.get((system, machine), f"{system}-{machine}")
24
27
 
25
28
 
29
+ # The shared connection's socket goes in ~/.ssh; ssh refuses to run when it cannot make it there.
30
+ _SHARED = () if os.name == "nt" else (
31
+ "-o", "ControlMaster=auto", "-o", "ControlPersist=60", "-o", "ControlPath=~/.ssh/ms_env-%C")
32
+
33
+
26
34
  class HostError(RuntimeError):
27
35
  pass
28
36
 
@@ -31,8 +39,10 @@ class Host:
31
39
  """A machine, known by what running a command on it gives back: `address` as `ssh` knows it (its
32
40
  keys, users and jumps are in `~/.ssh/config`), or None for this one."""
33
41
 
34
- def __init__(self, address: str | None = None, ssh=("ssh", "-o", "BatchMode=yes")):
42
+ def __init__(self, address: str | None = None, ssh=("ssh", "-o", "BatchMode=yes", *_SHARED)):
35
43
  self.address, self.ssh = address or None, tuple(ssh)
44
+ if self.address and _SHARED and _SHARED[-1] in self.ssh:
45
+ (Path.home() / ".ssh").mkdir(mode=0o700, exist_ok=True)
36
46
 
37
47
  def __repr__(self) -> str:
38
48
  return f"Host({self.address!r})"
@@ -0,0 +1,149 @@
1
+ """A short filesystem lock on an environment prefix, for the installs and removals of every client."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import logging
6
+ import os
7
+ import shlex
8
+ import socket
9
+ import subprocess
10
+ import time
11
+ from contextlib import contextmanager
12
+ from pathlib import Path
13
+
14
+ from ms_env.host import LOCAL, Host
15
+
16
+ log = logging.getLogger(__name__)
17
+
18
+ # The lock of the prefix `$p`, in `$lock`, with the parent's symlinks resolved as `Path.resolve` does here.
19
+ _LOCK_SH = ('p={prefix}; mkdir -p "$(dirname "$p")" || exit 1; '
20
+ 'parent=$(cd "$(dirname "$p")" && pwd -P) || exit 1; lock="$parent/$(basename "$p").ms-env-lock"; ')
21
+
22
+
23
+ class Locked(RuntimeError):
24
+ """The prefix is another client's now: `owner` is what it wrote (`client`, `pid`, `since`);
25
+ `unlock(prefix, host)` removes the lock once that client is known to have stopped."""
26
+
27
+ def __init__(self, prefix: str, host: Host, lock: str, owner: dict):
28
+ self.prefix, self.host, self.lock, self.owner = prefix, host, lock, owner
29
+ who = f"{owner.get('client', '?')} (pid {owner.get('pid', '?')}) since {owner.get('since', '?')}"
30
+ super().__init__(f"environment is locked by {who}: {lock}; once that client stopped, "
31
+ "`ms_env.unlock` removes the lock")
32
+
33
+
34
+ def _started(pid: int) -> str | None:
35
+ """When the process `pid` of this machine started, as `ps` says; "" when it does not run,
36
+ None when that cannot be told."""
37
+ try:
38
+ return subprocess.run(["ps", "-o", "lstart=", "-p", str(pid)], capture_output=True, text=True,
39
+ timeout=10).stdout.strip()
40
+ except (OSError, subprocess.TimeoutExpired):
41
+ return None
42
+
43
+
44
+ def _gone(owner: dict) -> bool:
45
+ """Whether the client that took a lock stopped. Only a client of this machine can be told:
46
+ by its pid, and by when that pid started, for a pid the system gave out again."""
47
+ if owner.get("client") != socket.gethostname() or not str(owner.get("pid", "")).isdigit():
48
+ return False
49
+ started = _started(int(owner["pid"]))
50
+ return started == "" or started is not None and owner.get("started", started) != started
51
+
52
+
53
+ def _owner(text: str) -> dict:
54
+ try:
55
+ owner = json.loads(text)
56
+ except ValueError:
57
+ return {} # killed between taking the lock and saying whose it is: not told apart
58
+ return owner if isinstance(owner, dict) else {}
59
+
60
+
61
+ def _local_lock(prefix: str) -> Path:
62
+ return Path(str(Path(prefix).expanduser().resolve()) + ".ms-env-lock")
63
+
64
+
65
+ def _drop(lock: str, host: Host, owner_text: str | None = None) -> None:
66
+ """Remove the lock, or, given `owner_text`, only while it is still the one with that owner."""
67
+ if host.address is None:
68
+ path = Path(lock, "owner.json")
69
+ if owner_text is None or (path.read_text() if path.is_file() else "") == owner_text:
70
+ path.unlink(missing_ok=True)
71
+ Path(lock).rmdir()
72
+ return
73
+ quoted = shlex.quote(lock)
74
+ drop = f"rm -f -- {quoted}/owner.json && rmdir -- {quoted}"
75
+ if owner_text is not None:
76
+ drop = f'if [ "$(cat {quoted}/owner.json 2>/dev/null)" = {shlex.quote(owner_text)} ]; then {drop}; fi'
77
+ host.run(drop, timeout=60)
78
+
79
+
80
+ def _take(prefix: str, host: Host, owner: str) -> tuple[str, str | None]:
81
+ """Take the lock: its path, and nothing; or, when it is somebody's, its path and what its owner wrote."""
82
+ if host.address is None:
83
+ lock = _local_lock(prefix)
84
+ lock.parent.mkdir(parents=True, exist_ok=True)
85
+ try:
86
+ lock.mkdir(mode=0o700)
87
+ except FileExistsError:
88
+ path = lock / "owner.json"
89
+ return str(lock), path.read_text() if path.is_file() else ""
90
+ try:
91
+ (lock / "owner.json").write_text(owner)
92
+ except BaseException:
93
+ lock.rmdir()
94
+ raise
95
+ return str(lock), None
96
+ said = host.run(_LOCK_SH.format(prefix=shlex.quote(prefix))
97
+ + 'if ! mkdir -m 700 "$lock" 2>/dev/null; then printf "locked\\n%s\\n" "$lock"; '
98
+ 'cat "$lock/owner.json" 2>/dev/null; exit 0; fi; '
99
+ + 'printf "%s" ' + shlex.quote(owner) + ' > "$lock/owner.json" || { rmdir "$lock"; exit 1; }; '
100
+ 'printf "free\\n%s\\n" "$lock"', timeout=60).decode()
101
+ state, lock, *rest = said.split("\n", 2)
102
+ return lock, ((rest or [""])[0] if state == "locked" else None)
103
+
104
+
105
+ @contextmanager
106
+ def prefix_lock(prefix: str, host: Host = LOCAL):
107
+ """Fail closed (`Locked`) when another client owns this prefix; no background service.
108
+
109
+ A lock left by a client of this machine that stopped (killed, crashed) is removed and taken
110
+ again. One of another machine is never guessed at, by age or else, as that is unsafe on a
111
+ cluster: `unlock` removes it once its owner is known to have stopped.
112
+ """
113
+ if not prefix or Path(prefix) == Path(prefix).parent:
114
+ raise ValueError("a lock requires a dedicated environment prefix")
115
+ owner = json.dumps({"client": socket.gethostname(), "pid": os.getpid(), "started": _started(os.getpid()) or "",
116
+ "since": time.strftime("%Y-%m-%d %H:%M:%S")})
117
+ lock, held = _take(prefix, host, owner)
118
+ if held is not None and _gone(_owner(held)):
119
+ # ponytail: two clients of this machine that find one stale lock at once can both drop it, and
120
+ # the second then drops the lock the first just took; the owner is checked again right before.
121
+ log.warning("removing the lock of %s left by a client that stopped: %s", prefix, held)
122
+ _drop(lock, host, held)
123
+ lock, held = _take(prefix, host, owner)
124
+ if held is not None:
125
+ raise Locked(prefix, host, lock, _owner(held))
126
+ try:
127
+ yield
128
+ finally:
129
+ _drop(lock, host)
130
+
131
+
132
+ def unlock(prefix: str, host: Host = LOCAL) -> dict:
133
+ """Remove the lock of `prefix`, whoever took it, and return what its owner wrote ({} for no lock).
134
+ Only once that owner is known to have stopped: else two clients change one environment."""
135
+ if host.address is None:
136
+ lock = _local_lock(prefix)
137
+ if not lock.is_dir():
138
+ return {}
139
+ path = lock / "owner.json"
140
+ held = path.read_text() if path.is_file() else ""
141
+ else:
142
+ said = host.run(_LOCK_SH.format(prefix=shlex.quote(prefix))
143
+ + '[ -d "$lock" ] || exit 0; printf "%s\\n" "$lock"; cat "$lock/owner.json" 2>/dev/null',
144
+ timeout=60).decode()
145
+ if not said:
146
+ return {}
147
+ lock, _, held = said.partition("\n")
148
+ _drop(str(lock), host)
149
+ return _owner(held)
@@ -74,7 +74,8 @@ from pathlib import Path, PurePath
74
74
  from typing import Any, Callable, Iterable
75
75
 
76
76
  from ms_env.steps import Step, StepFailed, find_environment_manager, run_step
77
- from ms_env.host import LOCAL, Host
77
+ from ms_env.host import LOCAL, Host, HostError
78
+ from ms_env.locking import prefix_lock
78
79
 
79
80
  READY = "ready"
80
81
  MISSING = "missing"
@@ -386,7 +387,8 @@ def _base(owner: str) -> str:
386
387
 
387
388
 
388
389
  def _executable(info: tarfile.TarInfo) -> tarfile.TarInfo:
389
- return info.replace(mode=0o755) # what Windows packs carries no mode; a download has to run there
390
+ # not deep: a deep copy takes the TarFile it points to along, with all it packed so far
391
+ return info.replace(mode=0o755, deep=False) # what Windows packs carries no mode; a download has to run there
390
392
 
391
393
 
392
394
  def _tar(paths: Iterable[str]) -> bytes:
@@ -397,8 +399,43 @@ def _tar(paths: Iterable[str]) -> bytes:
397
399
  return held.getvalue()
398
400
 
399
401
 
402
+ # `_fetch`, run there by the host's own python3: the same check and unpacking, with no ms_env there yet.
403
+ _FETCH_THERE = """import hashlib, os, shutil, sys, tempfile, urllib.request
404
+ url, sha256, to = sys.argv[1:4]
405
+ name = url.rsplit("/", 1)[-1]
406
+ with tempfile.TemporaryDirectory() as held:
407
+ file, digest = os.path.join(held, name), hashlib.sha256()
408
+ with urllib.request.urlopen(url, timeout=60) as got, open(file, "wb") as out:
409
+ for chunk in iter(lambda: got.read(1 << 20), b""):
410
+ digest.update(chunk)
411
+ out.write(chunk)
412
+ if digest.hexdigest() != sha256:
413
+ sys.exit(f"{url} is not the file the recipe names: sha256 {digest.hexdigest()}, not {sha256}")
414
+ if any(name.endswith(ext) for _, extensions, _ in shutil.get_unpack_formats() for ext in extensions):
415
+ try:
416
+ shutil.unpack_archive(file, to, **({} if name.endswith(".zip") else {"filter": "data"}))
417
+ except TypeError: # a python there older than the filter
418
+ shutil.unpack_archive(file, to)
419
+ else:
420
+ os.makedirs(os.path.dirname(to) or ".", exist_ok=True)
421
+ shutil.move(file, to)
422
+ os.chmod(to, 0o755)
423
+ """
424
+
425
+
426
+ def _fetch_there(step: RecipeStep, host: Host, emit: Callable[[str], None]) -> None:
427
+ """A download for a remote host: made there when the host reaches the URL, else made here and sent."""
428
+ emit(f"downloading {step.url} on {host.address}")
429
+ try:
430
+ host.run(shlex.join(["python3", "-", step.url, step.sha256, step.to]), data=_FETCH_THERE.encode(),
431
+ timeout=step.timeout)
432
+ except (HostError, subprocess.TimeoutExpired) as exc:
433
+ emit(f"{host.address} could not download it ({str(exc).splitlines()[-1][:200]}): sending it from here")
434
+ _fetch_and_send(step, host, emit)
435
+
436
+
400
437
  def _fetch_and_send(step: RecipeStep, host: Host, emit: Callable[[str], None]) -> None:
401
- """A download for a remote host: made and checked here, then sent, so the host needs no curl."""
438
+ """A download for a host that has no python3 or no way out: made and checked here, then sent."""
402
439
  with tempfile.TemporaryDirectory() as held:
403
440
  _fetch(step.url, step.sha256, str(Path(held, posixpath.basename(step.to))), emit)
404
441
  parent = shlex.quote(posixpath.dirname(step.to))
@@ -412,7 +449,7 @@ def _step(step: RecipeStep, label: str, host: Host) -> Step:
412
449
  return Step(label + step.name, run=step.local, timeout=step.timeout)
413
450
  if step.url:
414
451
  run = partial(_fetch, step.url, step.sha256, step.to) if host.address is None else \
415
- partial(_fetch_and_send, step, host)
452
+ partial(_fetch_there, step, host)
416
453
  return Step(label + step.name, run=run, timeout=step.timeout)
417
454
  if not step.send:
418
455
  return Step(label + step.name, argv=_argv_on(step.argv, host), timeout=step.timeout)
@@ -613,17 +650,18 @@ def remove(recipe: Recipe, host: Host = LOCAL) -> bool:
613
650
  current = _INSTALLS.get(key)
614
651
  if current is not None and not current.future.done():
615
652
  raise RuntimeError(f"{recipe.name} is being installed on {host.address or 'this machine'}.")
616
- if host.address is None:
617
- _prefix(recipe)
618
- if not (Path(recipe.prefix, STAMP).is_file() or Path(recipe.prefix + _INSTALLING).is_file()):
619
- return False
620
- _delete(recipe.prefix, recipe.prefix + _INSTALLING)
621
- return True
622
- script = (
623
- f'p={_prefix(recipe)}\n[ -f "$p/{STAMP}" ] || [ -f "$p{_INSTALLING}" ] || {{ echo no; exit 0; }}\n'
624
- f'rm -rf -- "$p" "$p{_INSTALLING}"'
625
- )
626
- return host.run(script, timeout=600).decode().strip() != "no"
653
+ with prefix_lock(recipe.prefix, host):
654
+ if host.address is None:
655
+ _prefix(recipe)
656
+ if not (Path(recipe.prefix, STAMP).is_file() or Path(recipe.prefix + _INSTALLING).is_file()):
657
+ return False
658
+ _delete(recipe.prefix, recipe.prefix + _INSTALLING)
659
+ return True
660
+ script = (
661
+ f'p={_prefix(recipe)}\n[ -f "$p/{STAMP}" ] || [ -f "$p{_INSTALLING}" ] || {{ echo no; exit 0; }}\n'
662
+ f'rm -rf -- "$p" "$p{_INSTALLING}"'
663
+ )
664
+ return host.run(script, timeout=600).decode().strip() != "no"
627
665
 
628
666
 
629
667
  # ----------------------------------------------------------------------------
@@ -635,10 +673,15 @@ class Install:
635
673
  key: str
636
674
  future: Future = field(default_factory=Future)
637
675
  lines: list[str] = field(default_factory=list) # live log, appended from the install thread
676
+ # Steps done of all it takes: the check, and then each step of an install when one is needed.
677
+ done: int = 0
678
+ total: int = 1
679
+ step: str = "" # the name of the step it is running, while it runs one
638
680
 
639
681
 
640
682
  _INSTALLS: dict[str, Install] = {}
641
683
  _LOCK = threading.Lock()
684
+ _CHANGING: dict[str, threading.Lock] = {} # by host: the checks run side by side, the installs one at a time
642
685
 
643
686
 
644
687
  def ensure(recipe: Recipe, host: Host = LOCAL, busy: Callable[[], str] | None = None) -> Install:
@@ -682,8 +725,18 @@ def _run_install(install: Install, recipe: Recipe, host: Host, busy: Callable[[]
682
725
  why = busy() if busy and owner.startswith("{") else "" # a stamp: an install of ours that ended
683
726
  if why:
684
727
  raise StepFailed(f"{recipe.name} on {where} is not as this recipe asks, and is left as it is: {why}")
685
- for step in recipe_steps(recipe, host, refresh=bool(recipe.base) and _base(owner) == recipe.base):
686
- run_step(step, install.lines.append)
728
+ steps = recipe_steps(recipe, host, refresh=bool(recipe.base) and _base(owner) == recipe.base)
729
+ install.done, install.total = 1, 1 + len(steps)
730
+ with _LOCK:
731
+ changing = _CHANGING.setdefault(host.address or "", threading.Lock())
732
+ # ponytail: one install at a time on a host (they share its micromamba and its package cache);
733
+ # the prefix lock keeps two clients of a shared home from changing the same environment.
734
+ with changing, prefix_lock(recipe.prefix, host):
735
+ for step in steps:
736
+ install.step = step.name
737
+ run_step(step, install.lines.append)
738
+ install.done += 1
739
+ install.step = ""
687
740
  state = status(recipe, host)
688
741
  if state != READY:
689
742
  raise StepFailed(f"{recipe.name}: every step ran but the tool is still {state} on {where}.")
@@ -691,6 +744,7 @@ def _run_install(install: Install, recipe: Recipe, host: Host, busy: Callable[[]
691
744
  install.lines.append(str(exc))
692
745
  install.future.set_exception(exc)
693
746
  else:
747
+ install.done = install.total
694
748
  install.future.set_result(None)
695
749
 
696
750
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ms_env
3
- Version: 0.3.2
3
+ Version: 0.4.0
4
4
  Summary: Environments for MolSuite apps: find or install external tools, on this machine or over ssh.
5
5
  Author: Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco
6
6
  License-Expression: MIT
@@ -125,7 +125,8 @@ as it is.
125
125
  `Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
126
126
  `ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
127
127
  logs in or the command fails at once, no credential is handled and nothing is left
128
- listening on the other side. A remote host is a POSIX machine: every command there runs through `sh`. This
128
+ listening on the other side. Outside Windows the commands to one host share one connection
129
+ (a socket in `~/.ssh`, closed a minute after the last command). A remote host is a POSIX machine: every command there runs through `sh`. This
129
130
  machine needs no `sh`: what ms_env does here it does in Python, Windows included.
130
131
 
131
132
  ## The environment of an app
@@ -12,6 +12,7 @@ docs/index.md
12
12
  src/ms_env/__init__.py
13
13
  src/ms_env/app.py
14
14
  src/ms_env/host.py
15
+ src/ms_env/locking.py
15
16
  src/ms_env/recipes.py
16
17
  src/ms_env/steps.py
17
18
  src/ms_env.egg-info/PKG-INFO
@@ -13,6 +13,7 @@
13
13
  "src/ms_env/__init__.py",
14
14
  "src/ms_env/app.py",
15
15
  "src/ms_env/host.py",
16
+ "src/ms_env/locking.py",
16
17
  "src/ms_env/recipes.py",
17
18
  "src/ms_env/steps.py",
18
19
  "tests/test_ms_env.py",
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "0.4.0",
3
+ "distance": 0,
4
+ "node": "g5da98367443b7261a98f74c9c3d2a6f68baf7b7e",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-11-02"
8
+ }
@@ -3,6 +3,8 @@ from __future__ import annotations
3
3
 
4
4
  import subprocess
5
5
  import dataclasses
6
+ import hashlib
7
+ import os
6
8
  import json
7
9
  import sys
8
10
  from pathlib import Path
@@ -57,7 +59,7 @@ def test_spec_binds_per_host(tmp_path):
57
59
  assert [step.name.removeprefix("mario@login.hpc: ") for step in steps] == [
58
60
  "prepare /opt/mine", "create the faketool environment", "pip install into faketool", "record the faketool install",
59
61
  ]
60
- assert steps[1].argv[:4] == ["ssh", "-o", "BatchMode=yes", "mario@login.hpc"]
62
+ assert steps[1].argv[:3] + [steps[1].argv[-2]] == ["ssh", "-o", "BatchMode=yes", "mario@login.hpc"]
61
63
  assert "/scratch/tools/micromamba" in steps[1].argv[-1] # bootstrap lands under the remote home
62
64
  assert steps[2].argv[-1].endswith("'extra pkg'") # one quoted remote command line
63
65
  assert remote.stamp == local.stamp # what the TOML asks for, wherever it is applied
@@ -130,6 +132,32 @@ def test_install_needs_no_project(tmp_path, monkeypatch):
130
132
  assert "boom detail" in broken.lines
131
133
 
132
134
 
135
+ def test_installs_on_one_host_take_turns_and_count_their_steps(tmp_path, monkeypatch):
136
+ import time
137
+
138
+ monkeypatch.setenv("PATH", "/usr/bin:/bin")
139
+ recipes = []
140
+ for name in ("a", "b"):
141
+ (tmp_path / name).mkdir()
142
+ recipes.append(dataclasses.replace(_plain_spec(tmp_path / name, slow=True), name=f"faketool-{name}"))
143
+ start = time.monotonic()
144
+ installs = [tools.ensure(recipe) for recipe in recipes]
145
+ running = set()
146
+ while not all(install.future.done() for install in installs):
147
+ running.update(install.step for install in installs)
148
+ time.sleep(0.05)
149
+ for install in installs:
150
+ install.future.result(timeout=30)
151
+ assert "wait" in running, running
152
+ assert all(install.step == "" for install in installs) # named while it runs, not after
153
+ assert time.monotonic() - start >= 2 # one `sleep 1` after the other: they share the host's micromamba
154
+ # the check, "prepare", the two steps of the recipe, the stamp
155
+ assert [(install.done, install.total) for install in installs] == [(5, 5), (5, 5)]
156
+ again = tools.ensure(recipes[0])
157
+ again.future.result(timeout=30)
158
+ assert (again.done, again.total) == (1, 1) # only the check: nothing to install
159
+
160
+
133
161
  def test_ms_env_updates_and_removes_what_it_installed(tmp_path, monkeypatch):
134
162
  monkeypatch.setenv("PATH", "/usr/bin:/bin")
135
163
  recipe = _plain_spec(tmp_path)
@@ -186,7 +214,8 @@ def test_the_environment_of_the_user_is_verified_and_never_changed(tmp_path, mon
186
214
 
187
215
  def test_a_host_runs_commands_and_says_why_one_failed():
188
216
  assert Host().run("cat; echo there", b"here ") == b"here there\n"
189
- assert Host("login.hpc").argv("ls") == ["ssh", "-o", "BatchMode=yes", "login.hpc", "ls"]
217
+ argv = Host("login.hpc").argv("ls")
218
+ assert argv[:3] + argv[-2:] == ["ssh", "-o", "BatchMode=yes", "login.hpc", "ls"]
190
219
  with pytest.raises(tools.HostError, match="this machine: no such thing"):
191
220
  Host().run("echo no such thing >&2; exit 3")
192
221
 
@@ -294,7 +323,7 @@ def test_a_download_is_picked_per_platform_checked_and_installed(tmp_path, monke
294
323
  tools.ToolSpec.load(tmp_path / "bad.toml")
295
324
 
296
325
 
297
- def test_an_archive_is_unpacked_and_a_remote_host_gets_the_download_sent(tmp_path, monkeypatch):
326
+ def test_an_archive_is_unpacked_and_a_remote_host_with_no_way_out_gets_the_download_sent(tmp_path, monkeypatch):
298
327
  import tarfile
299
328
  import zipfile
300
329
 
@@ -307,7 +336,13 @@ def test_an_archive_is_unpacked_and_a_remote_host_gets_the_download_sent(tmp_pat
307
336
  assert Path(recipe.prefix, "index.html").read_text() == "<html/>"
308
337
 
309
338
  sent: list[tuple[str, bytes]] = []
310
- monkeypatch.setattr(Host, "run", lambda self, command, data=b"", timeout=0: sent.append((command, data)) or b"")
339
+
340
+ def run(self, command, data=b"", timeout=0): # a host with no way out: what it cannot download is sent
341
+ if command.startswith("python3"):
342
+ raise tools.HostError("no route to host")
343
+ return sent.append((command, data)) or b""
344
+
345
+ monkeypatch.setattr(Host, "run", run)
311
346
  far = _download_spec(tmp_path, {"": _fake_exe(tmp_path / "release/faketool")}).recipe("/far/tools")
312
347
  run_step(tools.recipe_steps(far, Host("login.hpc"))[1])
313
348
  (command, data), = sent
@@ -343,3 +378,85 @@ def test_a_new_tag_on_the_same_code_builds_a_new_wheel(tmp_path):
343
378
  git("tag", "v0.2.0") # a release of the same code: its wheel has another version
344
379
  assert before.startswith("v0.1.0-1-g") and app._described(tmp_path).startswith("v0.2.0-0-g")
345
380
  assert app._described(tmp_path / "not-a-checkout") == ""
381
+
382
+
383
+ def test_a_folder_of_many_files_is_sent_as_a_tar_of_its_size(tmp_path):
384
+ import resource
385
+ import tarfile
386
+ from io import BytesIO
387
+
388
+ from ms_env.recipes import _tar
389
+
390
+ for i in range(300): # each file once took a deep copy of all packed before it: GBs by 100
391
+ (tmp_path / "pack" / f"sub{i % 10}").mkdir(parents=True, exist_ok=True)
392
+ (tmp_path / "pack" / f"sub{i % 10}" / f"f{i}.js").write_bytes(b"x" * 20_000)
393
+ before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
394
+ data = _tar([str(tmp_path / "pack")])
395
+ assert resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - before < 200 * 1024 # kB
396
+ with tarfile.open(fileobj=BytesIO(data)) as tar:
397
+ members = tar.getmembers()
398
+ assert len([m for m in members if m.isfile()]) == 300 and all(m.mode == 0o755 for m in members)
399
+
400
+
401
+ def test_a_host_downloads_for_itself_and_is_sent_the_file_only_when_it_cannot(tmp_path):
402
+ from ms_env.recipes import RecipeStep, _step
403
+
404
+ payload = tmp_path / "tool.bin"
405
+ payload.write_bytes(b"#!/bin/sh\necho hi\n")
406
+ digest = hashlib.sha256(payload.read_bytes()).hexdigest()
407
+
408
+ class Far(Host): # a "remote" host that is this machine; `blocked` is a host with no way out
409
+ blocked = False
410
+
411
+ def argv(self, command):
412
+ if self.blocked and command.startswith("python3"):
413
+ return ["sh", "-c", "echo 'no route to host' >&2; exit 1"]
414
+ return ["sh", "-c", command]
415
+
416
+ for blocked in (False, True):
417
+ host, said = Far("far"), []
418
+ host.blocked = blocked
419
+ to = tmp_path / f"there-{blocked}" / "tool"
420
+ _step(RecipeStep("get", (), 60.0, url=payload.as_uri(), sha256=digest, to=str(to)), "", host).run(said.append)
421
+ assert to.read_bytes() == payload.read_bytes() and os.access(to, os.X_OK)
422
+ assert any("sending it from here" in line for line in said) == blocked
423
+
424
+
425
+ @pytest.mark.parametrize("host", [tools.LOCAL, Host("far", ssh=("sh", "-c", 'exec sh -c "$2"', "_"))])
426
+ def test_a_prefix_is_changed_by_one_client_at_a_time(tmp_path, host):
427
+ prefix = str(tmp_path / "envs" / "tool")
428
+ with tools.prefix_lock(prefix, host):
429
+ assert (tmp_path / "envs" / "tool.ms-env-lock" / "owner.json").is_file()
430
+ with pytest.raises((RuntimeError, tools.HostError), match="locked"):
431
+ with tools.prefix_lock(prefix, host):
432
+ pass
433
+ assert not (tmp_path / "envs" / "tool.ms-env-lock").exists() # released: the next one gets it
434
+ recipe = tools.Recipe("tool", prefix, (), (), (), ())
435
+ with tools.prefix_lock(prefix, host), pytest.raises((RuntimeError, tools.HostError), match="locked"):
436
+ tools.remove(recipe, host)
437
+
438
+
439
+ @pytest.mark.parametrize("host", [tools.LOCAL, Host("far", ssh=("sh", "-c", 'exec sh -c "$2"', "_"))])
440
+ def test_a_lock_left_by_a_client_that_stopped_is_taken_only_when_this_machine_can_tell(tmp_path, host):
441
+ import json
442
+ import socket
443
+ import subprocess
444
+ import sys
445
+
446
+ prefix, lock = str(tmp_path / "tool"), tmp_path / "tool.ms-env-lock"
447
+ dead = subprocess.Popen([sys.executable, "-c", "pass"])
448
+ dead.wait()
449
+ for client, pid in ((socket.gethostname(), dead.pid), (socket.gethostname(), os.getpid())):
450
+ lock.mkdir()
451
+ (lock / "owner.json").write_text(json.dumps({"client": client, "pid": pid, "started": "then"}))
452
+ with tools.prefix_lock(prefix, host): # stopped, or this pid now another process: stale
453
+ assert json.loads((lock / "owner.json").read_text())["started"] != "then"
454
+ assert not lock.exists()
455
+ lock.mkdir()
456
+ (lock / "owner.json").write_text(json.dumps({"client": "elsewhere", "pid": 1, "since": "today"}))
457
+ with pytest.raises(tools.Locked, match="elsewhere") as said:
458
+ with tools.prefix_lock(prefix, host):
459
+ pass
460
+ assert said.value.owner["client"] == "elsewhere" and lock.exists() # another machine's: never guessed
461
+ assert tools.unlock(said.value.prefix, said.value.host)["client"] == "elsewhere" and not lock.exists()
462
+ assert tools.unlock(prefix, host) == {}
@@ -1,8 +0,0 @@
1
- {
2
- "tag": "0.3.2",
3
- "distance": 0,
4
- "node": "g3f644bbfcaf555dc7cedf971907597ad44ef3f75",
5
- "dirty": false,
6
- "branch": "HEAD",
7
- "node_date": "2026-10-28"
8
- }
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes