ms-env 0.3.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ms_env-0.3.2 → ms_env-0.4.0}/PKG-INFO +3 -2
- {ms_env-0.3.2 → ms_env-0.4.0}/README.md +2 -1
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/__init__.py +4 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/app.py +19 -12
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/host.py +11 -1
- ms_env-0.4.0/src/ms_env/locking.py +149 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/recipes.py +71 -17
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/PKG-INFO +3 -2
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/SOURCES.txt +1 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/scm_file_list.json +1 -0
- ms_env-0.4.0/src/ms_env.egg-info/scm_version.json +8 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/tests/test_ms_env.py +121 -4
- ms_env-0.3.2/src/ms_env.egg-info/scm_version.json +0 -8
- {ms_env-0.3.2 → ms_env-0.4.0}/.github/workflows/ci.yml +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/.github/workflows/docs.yml +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/.github/workflows/publish.yml +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/.gitignore +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/LICENSE +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/docs/api_reference.md +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/docs/guide.md +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/docs/index.md +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/pyproject.toml +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/setup.cfg +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env/steps.py +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/dependency_links.txt +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/requires.txt +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/src/ms_env.egg-info/top_level.txt +0 -0
- {ms_env-0.3.2 → ms_env-0.4.0}/zensical.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ms_env
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Environments for MolSuite apps: find or install external tools, on this machine or over ssh.
|
|
5
5
|
Author: Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco
|
|
6
6
|
License-Expression: MIT
|
|
@@ -125,7 +125,8 @@ as it is.
|
|
|
125
125
|
`Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
|
|
126
126
|
`ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
|
|
127
127
|
logs in or the command fails at once, no credential is handled and nothing is left
|
|
128
|
-
listening on the other side.
|
|
128
|
+
listening on the other side. Outside Windows the commands to one host share one connection
|
|
129
|
+
(a socket in `~/.ssh`, closed a minute after the last command). A remote host is a POSIX machine: every command there runs through `sh`. This
|
|
129
130
|
machine needs no `sh`: what ms_env does here it does in Python, Windows included.
|
|
130
131
|
|
|
131
132
|
## The environment of an app
|
|
@@ -100,7 +100,8 @@ as it is.
|
|
|
100
100
|
`Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
|
|
101
101
|
`ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
|
|
102
102
|
logs in or the command fails at once, no credential is handled and nothing is left
|
|
103
|
-
listening on the other side.
|
|
103
|
+
listening on the other side. Outside Windows the commands to one host share one connection
|
|
104
|
+
(a socket in `~/.ssh`, closed a minute after the last command). A remote host is a POSIX machine: every command there runs through `sh`. This
|
|
104
105
|
machine needs no `sh`: what ms_env does here it does in Python, Windows included.
|
|
105
106
|
|
|
106
107
|
## The environment of an app
|
|
@@ -22,6 +22,7 @@ from ms_env.recipes import (
|
|
|
22
22
|
)
|
|
23
23
|
from ms_env.steps import Step, StepFailed, find_environment_manager, run_step, run_steps
|
|
24
24
|
from ms_env.host import LOCAL, Host, HostError, conda_platform
|
|
25
|
+
from ms_env.locking import Locked, prefix_lock, unlock
|
|
25
26
|
|
|
26
27
|
__all__ = [
|
|
27
28
|
"Install",
|
|
@@ -48,4 +49,7 @@ __all__ = [
|
|
|
48
49
|
"run_steps",
|
|
49
50
|
"status",
|
|
50
51
|
"verify",
|
|
52
|
+
"prefix_lock",
|
|
53
|
+
"unlock",
|
|
54
|
+
"Locked",
|
|
51
55
|
]
|
|
@@ -71,23 +71,29 @@ def _wheel(root: Path, tag: str, cache: Path) -> Path:
|
|
|
71
71
|
return next(out.glob("*.whl"))
|
|
72
72
|
|
|
73
73
|
|
|
74
|
-
def _constraints(
|
|
75
|
-
"""`name==version` of what this environment holds: all of it,
|
|
74
|
+
def _constraints() -> tuple[str, str]:
|
|
75
|
+
"""`name==version` of what this environment holds: all of it, and what pip took from an index.
|
|
76
76
|
|
|
77
77
|
A local build is never pinned (no index has it), nor a name this environment has two
|
|
78
|
-
records of (which one runs is anyone's guess). What conda put here is pinned only
|
|
79
|
-
|
|
78
|
+
records of (which one runs is anyone's guess). What conda put here is pinned only in the
|
|
79
|
+
first: an index may lack that version, which is why the install has a second try.
|
|
80
80
|
"""
|
|
81
|
-
seen: dict[str, set[str]] = {}
|
|
82
|
-
for dist in metadata.distributions():
|
|
81
|
+
seen: dict[str, set[tuple[str, bool]]] = {}
|
|
82
|
+
for dist in metadata.distributions(): # one pass: reading every record is most of what this costs
|
|
83
83
|
name, installer = dist.metadata.get("Name"), (dist.read_text("INSTALLER") or "").strip()
|
|
84
84
|
if not name: # a broken leftover in site-packages
|
|
85
85
|
continue
|
|
86
86
|
local = "+" in dist.version or (installer != "conda" and dist.read_text("direct_url.json"))
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
87
|
+
seen.setdefault(re.sub(r"[-_.]+", "-", name).lower(), set()).add(
|
|
88
|
+
("" if local else dist.version, installer in ("pip", "uv")))
|
|
89
|
+
|
|
90
|
+
def pins(everything: bool) -> str:
|
|
91
|
+
versions = {name: {version if everything or by_pip else "" for version, by_pip in records}
|
|
92
|
+
for name, records in seen.items()}
|
|
93
|
+
return "\n".join(sorted(f"{name}=={next(iter(found))}" for name, found in versions.items()
|
|
94
|
+
if len(found) == 1 and next(iter(found)))) + "\n"
|
|
95
|
+
|
|
96
|
+
return pins(True), pins(False)
|
|
91
97
|
|
|
92
98
|
|
|
93
99
|
def app_recipe(
|
|
@@ -125,8 +131,9 @@ def app_recipe(
|
|
|
125
131
|
wanted.append(f"{code}/{wheels[-1].name}{extras or ''}")
|
|
126
132
|
cache.mkdir(parents=True, exist_ok=True)
|
|
127
133
|
constraints, loose = cache / "constraints.txt", cache / "constraints-pip.txt"
|
|
128
|
-
|
|
129
|
-
|
|
134
|
+
everything, by_pip = _constraints()
|
|
135
|
+
constraints.write_text(everything, encoding="utf-8")
|
|
136
|
+
loose.write_text(by_pip, encoding="utf-8")
|
|
130
137
|
|
|
131
138
|
pip = (f"{prefix}/bin/python", "-m", "pip", "install", "--progress-bar", "off")
|
|
132
139
|
# ponytail: one version the index lacks drops every pin of what conda installed here, not
|
|
@@ -2,9 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
No credential is handled here. `ssh` runs with `BatchMode=yes`: the keys and the agent of the
|
|
4
4
|
account log in or the command fails at once, and nothing is left listening on the other side.
|
|
5
|
+
Where `ssh` can (not Windows' OpenSSH), the commands to one host share one connection, kept a
|
|
6
|
+
minute after the last one: a check is then a few milliseconds instead of a new login.
|
|
5
7
|
"""
|
|
6
8
|
from __future__ import annotations
|
|
7
9
|
|
|
10
|
+
import os
|
|
8
11
|
import platform as _platform
|
|
9
12
|
import subprocess
|
|
10
13
|
from pathlib import Path
|
|
@@ -23,6 +26,11 @@ def conda_platform(system: str, machine: str) -> str:
|
|
|
23
26
|
return _PLATFORMS.get((system, machine), f"{system}-{machine}")
|
|
24
27
|
|
|
25
28
|
|
|
29
|
+
# The shared connection's socket goes in ~/.ssh; ssh refuses to run when it cannot make it there.
|
|
30
|
+
_SHARED = () if os.name == "nt" else (
|
|
31
|
+
"-o", "ControlMaster=auto", "-o", "ControlPersist=60", "-o", "ControlPath=~/.ssh/ms_env-%C")
|
|
32
|
+
|
|
33
|
+
|
|
26
34
|
class HostError(RuntimeError):
|
|
27
35
|
pass
|
|
28
36
|
|
|
@@ -31,8 +39,10 @@ class Host:
|
|
|
31
39
|
"""A machine, known by what running a command on it gives back: `address` as `ssh` knows it (its
|
|
32
40
|
keys, users and jumps are in `~/.ssh/config`), or None for this one."""
|
|
33
41
|
|
|
34
|
-
def __init__(self, address: str | None = None, ssh=("ssh", "-o", "BatchMode=yes")):
|
|
42
|
+
def __init__(self, address: str | None = None, ssh=("ssh", "-o", "BatchMode=yes", *_SHARED)):
|
|
35
43
|
self.address, self.ssh = address or None, tuple(ssh)
|
|
44
|
+
if self.address and _SHARED and _SHARED[-1] in self.ssh:
|
|
45
|
+
(Path.home() / ".ssh").mkdir(mode=0o700, exist_ok=True)
|
|
36
46
|
|
|
37
47
|
def __repr__(self) -> str:
|
|
38
48
|
return f"Host({self.address!r})"
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""A short filesystem lock on an environment prefix, for the installs and removals of every client."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import shlex
|
|
8
|
+
import socket
|
|
9
|
+
import subprocess
|
|
10
|
+
import time
|
|
11
|
+
from contextlib import contextmanager
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from ms_env.host import LOCAL, Host
|
|
15
|
+
|
|
16
|
+
log = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
# The lock of the prefix `$p`, in `$lock`, with the parent's symlinks resolved as `Path.resolve` does here.
|
|
19
|
+
_LOCK_SH = ('p={prefix}; mkdir -p "$(dirname "$p")" || exit 1; '
|
|
20
|
+
'parent=$(cd "$(dirname "$p")" && pwd -P) || exit 1; lock="$parent/$(basename "$p").ms-env-lock"; ')
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Locked(RuntimeError):
|
|
24
|
+
"""The prefix is another client's now: `owner` is what it wrote (`client`, `pid`, `since`);
|
|
25
|
+
`unlock(prefix, host)` removes the lock once that client is known to have stopped."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, prefix: str, host: Host, lock: str, owner: dict):
|
|
28
|
+
self.prefix, self.host, self.lock, self.owner = prefix, host, lock, owner
|
|
29
|
+
who = f"{owner.get('client', '?')} (pid {owner.get('pid', '?')}) since {owner.get('since', '?')}"
|
|
30
|
+
super().__init__(f"environment is locked by {who}: {lock}; once that client stopped, "
|
|
31
|
+
"`ms_env.unlock` removes the lock")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _started(pid: int) -> str | None:
|
|
35
|
+
"""When the process `pid` of this machine started, as `ps` says; "" when it does not run,
|
|
36
|
+
None when that cannot be told."""
|
|
37
|
+
try:
|
|
38
|
+
return subprocess.run(["ps", "-o", "lstart=", "-p", str(pid)], capture_output=True, text=True,
|
|
39
|
+
timeout=10).stdout.strip()
|
|
40
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _gone(owner: dict) -> bool:
|
|
45
|
+
"""Whether the client that took a lock stopped. Only a client of this machine can be told:
|
|
46
|
+
by its pid, and by when that pid started, for a pid the system gave out again."""
|
|
47
|
+
if owner.get("client") != socket.gethostname() or not str(owner.get("pid", "")).isdigit():
|
|
48
|
+
return False
|
|
49
|
+
started = _started(int(owner["pid"]))
|
|
50
|
+
return started == "" or started is not None and owner.get("started", started) != started
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _owner(text: str) -> dict:
|
|
54
|
+
try:
|
|
55
|
+
owner = json.loads(text)
|
|
56
|
+
except ValueError:
|
|
57
|
+
return {} # killed between taking the lock and saying whose it is: not told apart
|
|
58
|
+
return owner if isinstance(owner, dict) else {}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _local_lock(prefix: str) -> Path:
|
|
62
|
+
return Path(str(Path(prefix).expanduser().resolve()) + ".ms-env-lock")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _drop(lock: str, host: Host, owner_text: str | None = None) -> None:
|
|
66
|
+
"""Remove the lock, or, given `owner_text`, only while it is still the one with that owner."""
|
|
67
|
+
if host.address is None:
|
|
68
|
+
path = Path(lock, "owner.json")
|
|
69
|
+
if owner_text is None or (path.read_text() if path.is_file() else "") == owner_text:
|
|
70
|
+
path.unlink(missing_ok=True)
|
|
71
|
+
Path(lock).rmdir()
|
|
72
|
+
return
|
|
73
|
+
quoted = shlex.quote(lock)
|
|
74
|
+
drop = f"rm -f -- {quoted}/owner.json && rmdir -- {quoted}"
|
|
75
|
+
if owner_text is not None:
|
|
76
|
+
drop = f'if [ "$(cat {quoted}/owner.json 2>/dev/null)" = {shlex.quote(owner_text)} ]; then {drop}; fi'
|
|
77
|
+
host.run(drop, timeout=60)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _take(prefix: str, host: Host, owner: str) -> tuple[str, str | None]:
|
|
81
|
+
"""Take the lock: its path, and nothing; or, when it is somebody's, its path and what its owner wrote."""
|
|
82
|
+
if host.address is None:
|
|
83
|
+
lock = _local_lock(prefix)
|
|
84
|
+
lock.parent.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
try:
|
|
86
|
+
lock.mkdir(mode=0o700)
|
|
87
|
+
except FileExistsError:
|
|
88
|
+
path = lock / "owner.json"
|
|
89
|
+
return str(lock), path.read_text() if path.is_file() else ""
|
|
90
|
+
try:
|
|
91
|
+
(lock / "owner.json").write_text(owner)
|
|
92
|
+
except BaseException:
|
|
93
|
+
lock.rmdir()
|
|
94
|
+
raise
|
|
95
|
+
return str(lock), None
|
|
96
|
+
said = host.run(_LOCK_SH.format(prefix=shlex.quote(prefix))
|
|
97
|
+
+ 'if ! mkdir -m 700 "$lock" 2>/dev/null; then printf "locked\\n%s\\n" "$lock"; '
|
|
98
|
+
'cat "$lock/owner.json" 2>/dev/null; exit 0; fi; '
|
|
99
|
+
+ 'printf "%s" ' + shlex.quote(owner) + ' > "$lock/owner.json" || { rmdir "$lock"; exit 1; }; '
|
|
100
|
+
'printf "free\\n%s\\n" "$lock"', timeout=60).decode()
|
|
101
|
+
state, lock, *rest = said.split("\n", 2)
|
|
102
|
+
return lock, ((rest or [""])[0] if state == "locked" else None)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@contextmanager
|
|
106
|
+
def prefix_lock(prefix: str, host: Host = LOCAL):
|
|
107
|
+
"""Fail closed (`Locked`) when another client owns this prefix; no background service.
|
|
108
|
+
|
|
109
|
+
A lock left by a client of this machine that stopped (killed, crashed) is removed and taken
|
|
110
|
+
again. One of another machine is never guessed at, by age or else, as that is unsafe on a
|
|
111
|
+
cluster: `unlock` removes it once its owner is known to have stopped.
|
|
112
|
+
"""
|
|
113
|
+
if not prefix or Path(prefix) == Path(prefix).parent:
|
|
114
|
+
raise ValueError("a lock requires a dedicated environment prefix")
|
|
115
|
+
owner = json.dumps({"client": socket.gethostname(), "pid": os.getpid(), "started": _started(os.getpid()) or "",
|
|
116
|
+
"since": time.strftime("%Y-%m-%d %H:%M:%S")})
|
|
117
|
+
lock, held = _take(prefix, host, owner)
|
|
118
|
+
if held is not None and _gone(_owner(held)):
|
|
119
|
+
# ponytail: two clients of this machine that find one stale lock at once can both drop it, and
|
|
120
|
+
# the second then drops the lock the first just took; the owner is checked again right before.
|
|
121
|
+
log.warning("removing the lock of %s left by a client that stopped: %s", prefix, held)
|
|
122
|
+
_drop(lock, host, held)
|
|
123
|
+
lock, held = _take(prefix, host, owner)
|
|
124
|
+
if held is not None:
|
|
125
|
+
raise Locked(prefix, host, lock, _owner(held))
|
|
126
|
+
try:
|
|
127
|
+
yield
|
|
128
|
+
finally:
|
|
129
|
+
_drop(lock, host)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def unlock(prefix: str, host: Host = LOCAL) -> dict:
|
|
133
|
+
"""Remove the lock of `prefix`, whoever took it, and return what its owner wrote ({} for no lock).
|
|
134
|
+
Only once that owner is known to have stopped: else two clients change one environment."""
|
|
135
|
+
if host.address is None:
|
|
136
|
+
lock = _local_lock(prefix)
|
|
137
|
+
if not lock.is_dir():
|
|
138
|
+
return {}
|
|
139
|
+
path = lock / "owner.json"
|
|
140
|
+
held = path.read_text() if path.is_file() else ""
|
|
141
|
+
else:
|
|
142
|
+
said = host.run(_LOCK_SH.format(prefix=shlex.quote(prefix))
|
|
143
|
+
+ '[ -d "$lock" ] || exit 0; printf "%s\\n" "$lock"; cat "$lock/owner.json" 2>/dev/null',
|
|
144
|
+
timeout=60).decode()
|
|
145
|
+
if not said:
|
|
146
|
+
return {}
|
|
147
|
+
lock, _, held = said.partition("\n")
|
|
148
|
+
_drop(str(lock), host)
|
|
149
|
+
return _owner(held)
|
|
@@ -74,7 +74,8 @@ from pathlib import Path, PurePath
|
|
|
74
74
|
from typing import Any, Callable, Iterable
|
|
75
75
|
|
|
76
76
|
from ms_env.steps import Step, StepFailed, find_environment_manager, run_step
|
|
77
|
-
from ms_env.host import LOCAL, Host
|
|
77
|
+
from ms_env.host import LOCAL, Host, HostError
|
|
78
|
+
from ms_env.locking import prefix_lock
|
|
78
79
|
|
|
79
80
|
READY = "ready"
|
|
80
81
|
MISSING = "missing"
|
|
@@ -386,7 +387,8 @@ def _base(owner: str) -> str:
|
|
|
386
387
|
|
|
387
388
|
|
|
388
389
|
def _executable(info: tarfile.TarInfo) -> tarfile.TarInfo:
|
|
389
|
-
|
|
390
|
+
# not deep: a deep copy takes the TarFile it points to along, with all it packed so far
|
|
391
|
+
return info.replace(mode=0o755, deep=False) # what Windows packs carries no mode; a download has to run there
|
|
390
392
|
|
|
391
393
|
|
|
392
394
|
def _tar(paths: Iterable[str]) -> bytes:
|
|
@@ -397,8 +399,43 @@ def _tar(paths: Iterable[str]) -> bytes:
|
|
|
397
399
|
return held.getvalue()
|
|
398
400
|
|
|
399
401
|
|
|
402
|
+
# `_fetch`, run there by the host's own python3: the same check and unpacking, with no ms_env there yet.
|
|
403
|
+
_FETCH_THERE = """import hashlib, os, shutil, sys, tempfile, urllib.request
|
|
404
|
+
url, sha256, to = sys.argv[1:4]
|
|
405
|
+
name = url.rsplit("/", 1)[-1]
|
|
406
|
+
with tempfile.TemporaryDirectory() as held:
|
|
407
|
+
file, digest = os.path.join(held, name), hashlib.sha256()
|
|
408
|
+
with urllib.request.urlopen(url, timeout=60) as got, open(file, "wb") as out:
|
|
409
|
+
for chunk in iter(lambda: got.read(1 << 20), b""):
|
|
410
|
+
digest.update(chunk)
|
|
411
|
+
out.write(chunk)
|
|
412
|
+
if digest.hexdigest() != sha256:
|
|
413
|
+
sys.exit(f"{url} is not the file the recipe names: sha256 {digest.hexdigest()}, not {sha256}")
|
|
414
|
+
if any(name.endswith(ext) for _, extensions, _ in shutil.get_unpack_formats() for ext in extensions):
|
|
415
|
+
try:
|
|
416
|
+
shutil.unpack_archive(file, to, **({} if name.endswith(".zip") else {"filter": "data"}))
|
|
417
|
+
except TypeError: # a python there older than the filter
|
|
418
|
+
shutil.unpack_archive(file, to)
|
|
419
|
+
else:
|
|
420
|
+
os.makedirs(os.path.dirname(to) or ".", exist_ok=True)
|
|
421
|
+
shutil.move(file, to)
|
|
422
|
+
os.chmod(to, 0o755)
|
|
423
|
+
"""
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _fetch_there(step: RecipeStep, host: Host, emit: Callable[[str], None]) -> None:
|
|
427
|
+
"""A download for a remote host: made there when the host reaches the URL, else made here and sent."""
|
|
428
|
+
emit(f"downloading {step.url} on {host.address}")
|
|
429
|
+
try:
|
|
430
|
+
host.run(shlex.join(["python3", "-", step.url, step.sha256, step.to]), data=_FETCH_THERE.encode(),
|
|
431
|
+
timeout=step.timeout)
|
|
432
|
+
except (HostError, subprocess.TimeoutExpired) as exc:
|
|
433
|
+
emit(f"{host.address} could not download it ({str(exc).splitlines()[-1][:200]}): sending it from here")
|
|
434
|
+
_fetch_and_send(step, host, emit)
|
|
435
|
+
|
|
436
|
+
|
|
400
437
|
def _fetch_and_send(step: RecipeStep, host: Host, emit: Callable[[str], None]) -> None:
|
|
401
|
-
"""A download for a
|
|
438
|
+
"""A download for a host that has no python3 or no way out: made and checked here, then sent."""
|
|
402
439
|
with tempfile.TemporaryDirectory() as held:
|
|
403
440
|
_fetch(step.url, step.sha256, str(Path(held, posixpath.basename(step.to))), emit)
|
|
404
441
|
parent = shlex.quote(posixpath.dirname(step.to))
|
|
@@ -412,7 +449,7 @@ def _step(step: RecipeStep, label: str, host: Host) -> Step:
|
|
|
412
449
|
return Step(label + step.name, run=step.local, timeout=step.timeout)
|
|
413
450
|
if step.url:
|
|
414
451
|
run = partial(_fetch, step.url, step.sha256, step.to) if host.address is None else \
|
|
415
|
-
partial(
|
|
452
|
+
partial(_fetch_there, step, host)
|
|
416
453
|
return Step(label + step.name, run=run, timeout=step.timeout)
|
|
417
454
|
if not step.send:
|
|
418
455
|
return Step(label + step.name, argv=_argv_on(step.argv, host), timeout=step.timeout)
|
|
@@ -613,17 +650,18 @@ def remove(recipe: Recipe, host: Host = LOCAL) -> bool:
|
|
|
613
650
|
current = _INSTALLS.get(key)
|
|
614
651
|
if current is not None and not current.future.done():
|
|
615
652
|
raise RuntimeError(f"{recipe.name} is being installed on {host.address or 'this machine'}.")
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
653
|
+
with prefix_lock(recipe.prefix, host):
|
|
654
|
+
if host.address is None:
|
|
655
|
+
_prefix(recipe)
|
|
656
|
+
if not (Path(recipe.prefix, STAMP).is_file() or Path(recipe.prefix + _INSTALLING).is_file()):
|
|
657
|
+
return False
|
|
658
|
+
_delete(recipe.prefix, recipe.prefix + _INSTALLING)
|
|
659
|
+
return True
|
|
660
|
+
script = (
|
|
661
|
+
f'p={_prefix(recipe)}\n[ -f "$p/{STAMP}" ] || [ -f "$p{_INSTALLING}" ] || {{ echo no; exit 0; }}\n'
|
|
662
|
+
f'rm -rf -- "$p" "$p{_INSTALLING}"'
|
|
663
|
+
)
|
|
664
|
+
return host.run(script, timeout=600).decode().strip() != "no"
|
|
627
665
|
|
|
628
666
|
|
|
629
667
|
# ----------------------------------------------------------------------------
|
|
@@ -635,10 +673,15 @@ class Install:
|
|
|
635
673
|
key: str
|
|
636
674
|
future: Future = field(default_factory=Future)
|
|
637
675
|
lines: list[str] = field(default_factory=list) # live log, appended from the install thread
|
|
676
|
+
# Steps done of all it takes: the check, and then each step of an install when one is needed.
|
|
677
|
+
done: int = 0
|
|
678
|
+
total: int = 1
|
|
679
|
+
step: str = "" # the name of the step it is running, while it runs one
|
|
638
680
|
|
|
639
681
|
|
|
640
682
|
_INSTALLS: dict[str, Install] = {}
|
|
641
683
|
_LOCK = threading.Lock()
|
|
684
|
+
_CHANGING: dict[str, threading.Lock] = {} # by host: the checks run side by side, the installs one at a time
|
|
642
685
|
|
|
643
686
|
|
|
644
687
|
def ensure(recipe: Recipe, host: Host = LOCAL, busy: Callable[[], str] | None = None) -> Install:
|
|
@@ -682,8 +725,18 @@ def _run_install(install: Install, recipe: Recipe, host: Host, busy: Callable[[]
|
|
|
682
725
|
why = busy() if busy and owner.startswith("{") else "" # a stamp: an install of ours that ended
|
|
683
726
|
if why:
|
|
684
727
|
raise StepFailed(f"{recipe.name} on {where} is not as this recipe asks, and is left as it is: {why}")
|
|
685
|
-
|
|
686
|
-
|
|
728
|
+
steps = recipe_steps(recipe, host, refresh=bool(recipe.base) and _base(owner) == recipe.base)
|
|
729
|
+
install.done, install.total = 1, 1 + len(steps)
|
|
730
|
+
with _LOCK:
|
|
731
|
+
changing = _CHANGING.setdefault(host.address or "", threading.Lock())
|
|
732
|
+
# ponytail: one install at a time on a host (they share its micromamba and its package cache);
|
|
733
|
+
# the prefix lock keeps two clients of a shared home from changing the same environment.
|
|
734
|
+
with changing, prefix_lock(recipe.prefix, host):
|
|
735
|
+
for step in steps:
|
|
736
|
+
install.step = step.name
|
|
737
|
+
run_step(step, install.lines.append)
|
|
738
|
+
install.done += 1
|
|
739
|
+
install.step = ""
|
|
687
740
|
state = status(recipe, host)
|
|
688
741
|
if state != READY:
|
|
689
742
|
raise StepFailed(f"{recipe.name}: every step ran but the tool is still {state} on {where}.")
|
|
@@ -691,6 +744,7 @@ def _run_install(install: Install, recipe: Recipe, host: Host, busy: Callable[[]
|
|
|
691
744
|
install.lines.append(str(exc))
|
|
692
745
|
install.future.set_exception(exc)
|
|
693
746
|
else:
|
|
747
|
+
install.done = install.total
|
|
694
748
|
install.future.set_result(None)
|
|
695
749
|
|
|
696
750
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ms_env
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Environments for MolSuite apps: find or install external tools, on this machine or over ssh.
|
|
5
5
|
Author: Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco
|
|
6
6
|
License-Expression: MIT
|
|
@@ -125,7 +125,8 @@ as it is.
|
|
|
125
125
|
`Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
|
|
126
126
|
`ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
|
|
127
127
|
logs in or the command fails at once, no credential is handled and nothing is left
|
|
128
|
-
listening on the other side.
|
|
128
|
+
listening on the other side. Outside Windows the commands to one host share one connection
|
|
129
|
+
(a socket in `~/.ssh`, closed a minute after the last command). A remote host is a POSIX machine: every command there runs through `sh`. This
|
|
129
130
|
machine needs no `sh`: what ms_env does here it does in Python, Windows included.
|
|
130
131
|
|
|
131
132
|
## The environment of an app
|
|
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|
|
3
3
|
|
|
4
4
|
import subprocess
|
|
5
5
|
import dataclasses
|
|
6
|
+
import hashlib
|
|
7
|
+
import os
|
|
6
8
|
import json
|
|
7
9
|
import sys
|
|
8
10
|
from pathlib import Path
|
|
@@ -57,7 +59,7 @@ def test_spec_binds_per_host(tmp_path):
|
|
|
57
59
|
assert [step.name.removeprefix("mario@login.hpc: ") for step in steps] == [
|
|
58
60
|
"prepare /opt/mine", "create the faketool environment", "pip install into faketool", "record the faketool install",
|
|
59
61
|
]
|
|
60
|
-
assert steps[1].argv[:
|
|
62
|
+
assert steps[1].argv[:3] + [steps[1].argv[-2]] == ["ssh", "-o", "BatchMode=yes", "mario@login.hpc"]
|
|
61
63
|
assert "/scratch/tools/micromamba" in steps[1].argv[-1] # bootstrap lands under the remote home
|
|
62
64
|
assert steps[2].argv[-1].endswith("'extra pkg'") # one quoted remote command line
|
|
63
65
|
assert remote.stamp == local.stamp # what the TOML asks for, wherever it is applied
|
|
@@ -130,6 +132,32 @@ def test_install_needs_no_project(tmp_path, monkeypatch):
|
|
|
130
132
|
assert "boom detail" in broken.lines
|
|
131
133
|
|
|
132
134
|
|
|
135
|
+
def test_installs_on_one_host_take_turns_and_count_their_steps(tmp_path, monkeypatch):
|
|
136
|
+
import time
|
|
137
|
+
|
|
138
|
+
monkeypatch.setenv("PATH", "/usr/bin:/bin")
|
|
139
|
+
recipes = []
|
|
140
|
+
for name in ("a", "b"):
|
|
141
|
+
(tmp_path / name).mkdir()
|
|
142
|
+
recipes.append(dataclasses.replace(_plain_spec(tmp_path / name, slow=True), name=f"faketool-{name}"))
|
|
143
|
+
start = time.monotonic()
|
|
144
|
+
installs = [tools.ensure(recipe) for recipe in recipes]
|
|
145
|
+
running = set()
|
|
146
|
+
while not all(install.future.done() for install in installs):
|
|
147
|
+
running.update(install.step for install in installs)
|
|
148
|
+
time.sleep(0.05)
|
|
149
|
+
for install in installs:
|
|
150
|
+
install.future.result(timeout=30)
|
|
151
|
+
assert "wait" in running, running
|
|
152
|
+
assert all(install.step == "" for install in installs) # named while it runs, not after
|
|
153
|
+
assert time.monotonic() - start >= 2 # one `sleep 1` after the other: they share the host's micromamba
|
|
154
|
+
# the check, "prepare", the two steps of the recipe, the stamp
|
|
155
|
+
assert [(install.done, install.total) for install in installs] == [(5, 5), (5, 5)]
|
|
156
|
+
again = tools.ensure(recipes[0])
|
|
157
|
+
again.future.result(timeout=30)
|
|
158
|
+
assert (again.done, again.total) == (1, 1) # only the check: nothing to install
|
|
159
|
+
|
|
160
|
+
|
|
133
161
|
def test_ms_env_updates_and_removes_what_it_installed(tmp_path, monkeypatch):
|
|
134
162
|
monkeypatch.setenv("PATH", "/usr/bin:/bin")
|
|
135
163
|
recipe = _plain_spec(tmp_path)
|
|
@@ -186,7 +214,8 @@ def test_the_environment_of_the_user_is_verified_and_never_changed(tmp_path, mon
|
|
|
186
214
|
|
|
187
215
|
def test_a_host_runs_commands_and_says_why_one_failed():
|
|
188
216
|
assert Host().run("cat; echo there", b"here ") == b"here there\n"
|
|
189
|
-
|
|
217
|
+
argv = Host("login.hpc").argv("ls")
|
|
218
|
+
assert argv[:3] + argv[-2:] == ["ssh", "-o", "BatchMode=yes", "login.hpc", "ls"]
|
|
190
219
|
with pytest.raises(tools.HostError, match="this machine: no such thing"):
|
|
191
220
|
Host().run("echo no such thing >&2; exit 3")
|
|
192
221
|
|
|
@@ -294,7 +323,7 @@ def test_a_download_is_picked_per_platform_checked_and_installed(tmp_path, monke
|
|
|
294
323
|
tools.ToolSpec.load(tmp_path / "bad.toml")
|
|
295
324
|
|
|
296
325
|
|
|
297
|
-
def
|
|
326
|
+
def test_an_archive_is_unpacked_and_a_remote_host_with_no_way_out_gets_the_download_sent(tmp_path, monkeypatch):
|
|
298
327
|
import tarfile
|
|
299
328
|
import zipfile
|
|
300
329
|
|
|
@@ -307,7 +336,13 @@ def test_an_archive_is_unpacked_and_a_remote_host_gets_the_download_sent(tmp_pat
|
|
|
307
336
|
assert Path(recipe.prefix, "index.html").read_text() == "<html/>"
|
|
308
337
|
|
|
309
338
|
sent: list[tuple[str, bytes]] = []
|
|
310
|
-
|
|
339
|
+
|
|
340
|
+
def run(self, command, data=b"", timeout=0): # a host with no way out: what it cannot download is sent
|
|
341
|
+
if command.startswith("python3"):
|
|
342
|
+
raise tools.HostError("no route to host")
|
|
343
|
+
return sent.append((command, data)) or b""
|
|
344
|
+
|
|
345
|
+
monkeypatch.setattr(Host, "run", run)
|
|
311
346
|
far = _download_spec(tmp_path, {"": _fake_exe(tmp_path / "release/faketool")}).recipe("/far/tools")
|
|
312
347
|
run_step(tools.recipe_steps(far, Host("login.hpc"))[1])
|
|
313
348
|
(command, data), = sent
|
|
@@ -343,3 +378,85 @@ def test_a_new_tag_on_the_same_code_builds_a_new_wheel(tmp_path):
|
|
|
343
378
|
git("tag", "v0.2.0") # a release of the same code: its wheel has another version
|
|
344
379
|
assert before.startswith("v0.1.0-1-g") and app._described(tmp_path).startswith("v0.2.0-0-g")
|
|
345
380
|
assert app._described(tmp_path / "not-a-checkout") == ""
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def test_a_folder_of_many_files_is_sent_as_a_tar_of_its_size(tmp_path):
|
|
384
|
+
import resource
|
|
385
|
+
import tarfile
|
|
386
|
+
from io import BytesIO
|
|
387
|
+
|
|
388
|
+
from ms_env.recipes import _tar
|
|
389
|
+
|
|
390
|
+
for i in range(300): # each file once took a deep copy of all packed before it: GBs by 100
|
|
391
|
+
(tmp_path / "pack" / f"sub{i % 10}").mkdir(parents=True, exist_ok=True)
|
|
392
|
+
(tmp_path / "pack" / f"sub{i % 10}" / f"f{i}.js").write_bytes(b"x" * 20_000)
|
|
393
|
+
before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
394
|
+
data = _tar([str(tmp_path / "pack")])
|
|
395
|
+
assert resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - before < 200 * 1024 # kB
|
|
396
|
+
with tarfile.open(fileobj=BytesIO(data)) as tar:
|
|
397
|
+
members = tar.getmembers()
|
|
398
|
+
assert len([m for m in members if m.isfile()]) == 300 and all(m.mode == 0o755 for m in members)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def test_a_host_downloads_for_itself_and_is_sent_the_file_only_when_it_cannot(tmp_path):
|
|
402
|
+
from ms_env.recipes import RecipeStep, _step
|
|
403
|
+
|
|
404
|
+
payload = tmp_path / "tool.bin"
|
|
405
|
+
payload.write_bytes(b"#!/bin/sh\necho hi\n")
|
|
406
|
+
digest = hashlib.sha256(payload.read_bytes()).hexdigest()
|
|
407
|
+
|
|
408
|
+
class Far(Host): # a "remote" host that is this machine; `blocked` is a host with no way out
|
|
409
|
+
blocked = False
|
|
410
|
+
|
|
411
|
+
def argv(self, command):
|
|
412
|
+
if self.blocked and command.startswith("python3"):
|
|
413
|
+
return ["sh", "-c", "echo 'no route to host' >&2; exit 1"]
|
|
414
|
+
return ["sh", "-c", command]
|
|
415
|
+
|
|
416
|
+
for blocked in (False, True):
|
|
417
|
+
host, said = Far("far"), []
|
|
418
|
+
host.blocked = blocked
|
|
419
|
+
to = tmp_path / f"there-{blocked}" / "tool"
|
|
420
|
+
_step(RecipeStep("get", (), 60.0, url=payload.as_uri(), sha256=digest, to=str(to)), "", host).run(said.append)
|
|
421
|
+
assert to.read_bytes() == payload.read_bytes() and os.access(to, os.X_OK)
|
|
422
|
+
assert any("sending it from here" in line for line in said) == blocked
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
@pytest.mark.parametrize("host", [tools.LOCAL, Host("far", ssh=("sh", "-c", 'exec sh -c "$2"', "_"))])
|
|
426
|
+
def test_a_prefix_is_changed_by_one_client_at_a_time(tmp_path, host):
|
|
427
|
+
prefix = str(tmp_path / "envs" / "tool")
|
|
428
|
+
with tools.prefix_lock(prefix, host):
|
|
429
|
+
assert (tmp_path / "envs" / "tool.ms-env-lock" / "owner.json").is_file()
|
|
430
|
+
with pytest.raises((RuntimeError, tools.HostError), match="locked"):
|
|
431
|
+
with tools.prefix_lock(prefix, host):
|
|
432
|
+
pass
|
|
433
|
+
assert not (tmp_path / "envs" / "tool.ms-env-lock").exists() # released: the next one gets it
|
|
434
|
+
recipe = tools.Recipe("tool", prefix, (), (), (), ())
|
|
435
|
+
with tools.prefix_lock(prefix, host), pytest.raises((RuntimeError, tools.HostError), match="locked"):
|
|
436
|
+
tools.remove(recipe, host)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
@pytest.mark.parametrize("host", [tools.LOCAL, Host("far", ssh=("sh", "-c", 'exec sh -c "$2"', "_"))])
|
|
440
|
+
def test_a_lock_left_by_a_client_that_stopped_is_taken_only_when_this_machine_can_tell(tmp_path, host):
|
|
441
|
+
import json
|
|
442
|
+
import socket
|
|
443
|
+
import subprocess
|
|
444
|
+
import sys
|
|
445
|
+
|
|
446
|
+
prefix, lock = str(tmp_path / "tool"), tmp_path / "tool.ms-env-lock"
|
|
447
|
+
dead = subprocess.Popen([sys.executable, "-c", "pass"])
|
|
448
|
+
dead.wait()
|
|
449
|
+
for client, pid in ((socket.gethostname(), dead.pid), (socket.gethostname(), os.getpid())):
|
|
450
|
+
lock.mkdir()
|
|
451
|
+
(lock / "owner.json").write_text(json.dumps({"client": client, "pid": pid, "started": "then"}))
|
|
452
|
+
with tools.prefix_lock(prefix, host): # stopped, or this pid now another process: stale
|
|
453
|
+
assert json.loads((lock / "owner.json").read_text())["started"] != "then"
|
|
454
|
+
assert not lock.exists()
|
|
455
|
+
lock.mkdir()
|
|
456
|
+
(lock / "owner.json").write_text(json.dumps({"client": "elsewhere", "pid": 1, "since": "today"}))
|
|
457
|
+
with pytest.raises(tools.Locked, match="elsewhere") as said:
|
|
458
|
+
with tools.prefix_lock(prefix, host):
|
|
459
|
+
pass
|
|
460
|
+
assert said.value.owner["client"] == "elsewhere" and lock.exists() # another machine's: never guessed
|
|
461
|
+
assert tools.unlock(said.value.prefix, said.value.host)["client"] == "elsewhere" and not lock.exists()
|
|
462
|
+
assert tools.unlock(prefix, host) == {}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|