ms-env 0.2.0__py3-none-any.whl

ms_env/__init__.py

@@ -0,0 +1,50 @@
+"""Environments for MolSuite apps: a recipe says how to tell that a tool works and how to
+install it, a host says where, and `ensure` applies one to the other.
+
+The lowest layer of MolSuite: the standard library, no Qt, and nothing of `ms_flow`.
+"""
+from ms_env.app import app_recipe
+from ms_env.recipes import (
+    MISSING,
+    READY,
+    STAMP,
+    UNREACHABLE,
+    Install,
+    Recipe,
+    RecipeStep,
+    ToolSpec,
+    ensure,
+    locate,
+    recipe_steps,
+    remove,
+    status,
+    verify,
+)
+from ms_env.steps import Step, StepFailed, find_environment_manager, run_step, run_steps
+from ms_env.host import LOCAL, Host, HostError
+
+__all__ = [
+    "Install",
+    "LOCAL",
+    "MISSING",
+    "READY",
+    "Recipe",
+    "RecipeStep",
+    "STAMP",
+    "Step",
+    "StepFailed",
+    "Host",
+    "HostError",
+    "ToolSpec",
+    "UNREACHABLE",
+    "app_recipe",
+    "ensure",
+    "find_environment_manager",
+    "locate",
+    "recipe_steps",
+    "remove",
+    "run_step",
+    "run_steps",
+    "status",
+    "verify",
+]

ms_env/app.py

@@ -0,0 +1,165 @@
+"""The environment of an app on a host: the Python that runs here, with the same versions of
+what this one has installed.
+
+Nothing is copied but the code that runs here from a source checkout, which travels as wheels
+built from it. Everything else the host downloads, pinned to what this environment holds, so
+the two may be different systems.
+"""
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import platform
+import re
+import shutil
+import subprocess
+import sys
+from importlib import metadata
+from pathlib import Path
+from typing import Iterable
+
+from ms_env.recipes import Recipe, RecipeStep, _env_steps
+
+_CODE = ".ms_env.d"  # inside the prefix: the wheels and the constraints of the last install
+
+
+def _checkout(module: str) -> Path | None:
+    """The project `module` is imported from, when that is a source checkout and not an install."""
+    spec = importlib.util.find_spec(module)
+    if spec is None or not spec.origin:
+        raise ModuleNotFoundError(f"{module} does not import here, so no host can be given the same.")
+    origin = Path(spec.origin).resolve()
+    if {"site-packages", "dist-packages"} & set(origin.parts):
+        return None
+    return next((parent for parent in origin.parents if (parent / "pyproject.toml").is_file()), None)
+
+
+def _tree_hash(root: Path, module: str) -> str:
+    digest = hashlib.sha256((root / "pyproject.toml").read_bytes())
+    package = Path(importlib.util.find_spec(module).origin).resolve().parent
+    for path in sorted(package.rglob("*")):
+        if path.is_file() and "__pycache__" not in path.parts:
+            digest.update(str(path.relative_to(package)).encode() + path.read_bytes())
+    return digest.hexdigest()[:16]
+
+
+def _wheel(root: Path, tag: str, cache: Path) -> Path:
+    """The wheel of the checkout at `root` as it is now, built once for each state of its code."""
+    out = cache / f"{root.name}-{tag}"
+    if not list(out.glob("*.whl")):
+        for old in cache.glob(f"{root.name}-*"):
+            shutil.rmtree(old, ignore_errors=True)
+        done = subprocess.run(
+            [sys.executable, "-m", "pip", "wheel", "--no-deps", "--quiet", "--wheel-dir", str(out), str(root)],
+            capture_output=True, text=True, check=False,
+        )
+        if done.returncode or not list(out.glob("*.whl")):
+            raise RuntimeError(f"no wheel could be built from {root}:\n{done.stderr.strip()[-2000:]}")
+    return next(out.glob("*.whl"))
+
+
+def _constraints(everything: bool) -> str:
+    """`name==version` of what this environment holds: all of it, or what pip took from an index.
+
+    A local build is never pinned (no index has it), nor a name this environment has two
+    records of (which one runs is anyone's guess). What conda put here is pinned only when
+    `everything`: an index may lack that version, which is why the install has a second try.
+    """
+    seen: dict[str, set[str]] = {}
+    for dist in metadata.distributions():
+        name, installer = dist.metadata.get("Name"), (dist.read_text("INSTALLER") or "").strip()
+        if not name:  # a broken leftover in site-packages
+            continue
+        local = "+" in dist.version or (installer != "conda" and dist.read_text("direct_url.json"))
+        pin = not local and (everything or installer in ("pip", "uv"))
+        seen.setdefault(re.sub(r"[-_.]+", "-", name).lower(), set()).add(dist.version if pin else "")
+    pins = (f"{name}=={next(iter(versions))}" for name, versions in seen.items() if len(versions) == 1)
+    return "\n".join(sorted(pin for pin in pins if not pin.endswith("=="))) + "\n"
+
+
+def app_recipe(
+    name: str,
+    requirements: Iterable[str],
+    imports: Iterable[str],
+    *,
+    prefix: str,
+    tools_home: str,
+    cache: Path | str,
+    manager: str = "",
+) -> Recipe:
+    """The recipe that gives a host the environment `requirements` run in here.
+
+    `requirements` are the packages of the app, each `name` or `name[extras]`: one imported
+    from a source checkout goes as a wheel of its code as it is now, one that is installed is
+    asked for in the version it has. `imports` are the modules that have to import there for
+    the environment to count as usable. `prefix` and `tools_home` are paths on the host (see
+    `Host.home`); `cache` is a directory of this machine for the wheels.
+
+    The environment is updated where it stands for as long as the Python is the same. Builds
+    wheels the first time, and after the code changes: keep it off UI threads.
+    """
+    prefix, cache = str(prefix).rstrip("/"), Path(cache)
+    code, wheels, wanted, tags = f"{prefix}/{_CODE}", [], [], []
+    for requirement in requirements:
+        dist, extras = re.fullmatch(r"\s*([\w.-]+)\s*(\[[^]]*])?\s*", requirement).groups()
+        module = re.sub(r"[-.]+", "_", dist).lower()
+        root = _checkout(module)
+        if root is None:
+            wanted.append(f"{dist}{extras or ''}=={metadata.version(dist)}")
+            continue
+        tags.append(_tree_hash(root, module))
+        wheels.append(_wheel(root, tags[-1], cache))
+        wanted.append(f"{code}/{wheels[-1].name}{extras or ''}")
+    cache.mkdir(parents=True, exist_ok=True)
+    constraints, loose = cache / "constraints.txt", cache / "constraints-pip.txt"
+    constraints.write_text(_constraints(True), encoding="utf-8")
+    loose.write_text(_constraints(False), encoding="utf-8")
+
+    pip = (f"{prefix}/bin/python", "-m", "pip", "install", "--progress-bar", "off")
+    # ponytail: one version the index lacks drops every pin of what conda installed here, not
+    # just that one. Drop them one by one if a host ever ends up too far from this environment.
+    pinned = (
+        'pip() { "$0" -m pip install --progress-bar off --constraint "$1" "${@:2}"; }; d=$1; shift; '
+        'pip "$d/constraints.txt" "$@" || '
+        '{ echo "not every version here can be had there: pinning what pip installed only"; '
+        'pip "$d/constraints-pip.txt" "$@"; }'
+    )
+    refresh = [
+        RecipeStep(
+            f"send the code of {name}",
+            ("sh", "-c", 'rm -rf "$0" && mkdir -p "$0" && tar -x -C "$0"', code),
+            600.0,
+            send=(str(constraints), str(loose), *map(str, wheels)),
+        ),
+        RecipeStep(
+            f"install {name} and what it needs",
+            ("bash", "-c", pinned, f"{prefix}/bin/python", code, *wanted),
+            3600.0,
+        ),
+    ]
+    if wheels:  # the version of a checkout does not change with its code: pip would keep the old one
+        refresh.append(
+            RecipeStep(
+                f"install the code of {name} as it is here",
+                (*pip, "--no-deps", "--force-reinstall", *(f"{code}/{wheel.name}" for wheel in wheels)),
+                600.0,
+            )
+        )
+    conda = (f"python={platform.python_version()}", "pip")
+    values = {"prefix": prefix, "tools_home": str(tools_home).rstrip("/")}
+    short = lambda *parts: hashlib.sha256(repr(parts).encode()).hexdigest()[:16]  # noqa: E731
+    return Recipe(
+        name=name,
+        prefix=prefix,
+        candidates=(f"{prefix}/bin/python",),
+        commands=(),
+        check=("{exe}", "-c", "import " + ", ".join(imports)),
+        steps=(*_env_steps(name, conda, (), ("conda-forge",), values, manager), *refresh),
+        conda=conda,
+        stamp=short(conda, wanted, tags, constraints.read_text(encoding="utf-8")),
+        refresh=tuple(refresh),
+        base=short(conda),
+    )
+
+
+__all__ = ["app_recipe"]

ms_env/host.py

@@ -0,0 +1,50 @@
+"""A host: where commands run. This machine, or another reached with the system's `ssh`.
+
+No credential is handled here. `ssh` runs with `BatchMode=yes`: the keys and the agent of the
+account log in or the command fails at once, and nothing is left listening on the other side.
+"""
+from __future__ import annotations
+
+import subprocess
+
+
+class HostError(RuntimeError):
+    pass
+
+
+class Host:
+    """A machine, known by what running a command on it gives back: `address` as `ssh` knows it (its
+    keys, users and jumps are in `~/.ssh/config`), or None for this one."""
+
+    def __init__(self, address: str | None = None, ssh=("ssh", "-o", "BatchMode=yes")):
+        self.address, self.ssh = address or None, tuple(ssh)
+
+    def __repr__(self) -> str:
+        return f"Host({self.address!r})"
+
+    def argv(self, command: str) -> list[str]:
+        """What to execute here for `sh` to run `command` there."""
+        return ["sh", "-c", command] if self.address is None else [*self.ssh, self.address, command]
+
+    def run(self, command: str, data=b"", timeout: float = 600) -> bytes:
+        """What `command` printed there, given `data` (bytes, or an open file) to read."""
+        feed = {"input": data} if isinstance(data, bytes) else {"stdin": data}
+        done = subprocess.run(self.argv(command), **feed, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                              timeout=timeout)
+        if done.returncode:
+            said = done.stderr.decode(errors="replace").strip()
+            if "Permission denied" in said:  # BatchMode: ssh does not ask, and here nobody could answer
+                said += f"\nssh has to log in to {self.address} with a key, without asking: see ssh-copy-id"
+            if "Host key verification failed" in said:  # the same: nobody here to say yes to a new key
+                said += f"\nssh does not know {self.address} yet: log in to it once by hand (`ssh {self.address}`)"
+            raise HostError(f"`{command[:200]}` on {self.address or 'this machine'}: {said or done.returncode}")
+        return done.stdout
+
+    def home(self) -> str:
+        """The home of the account there: where a path of ours on that machine starts."""
+        return self.run('printf %s "$HOME"', timeout=60).decode()
+
+
+LOCAL = Host()
+
+__all__ = ["LOCAL", "Host", "HostError"]

ms_env/recipes.py

@@ -0,0 +1,481 @@
+"""External tools: find one that already works, or install it -- driven by a TOML file.
+
+The app ships one TOML per tool: where to look for it, how to tell it works, how to
+install it. A recipe is that TOML bound to one host (see ``ms_env.host``): this
+machine, or a host over ssh. ``status`` says whether the tool works there and ``ensure``
+installs it when it does not, on a daemon thread that belongs to the app, not to a
+project (none has to be open).
+
+Lookup on a host, first candidate whose check exits 0 wins: paths the user
+registered, the managed install under ``tools_home``, then ``PATH``.
+
+Whose a prefix is decides what happens to it. ``ensure`` stamps what it installs: a
+stamped prefix is created again when the recipe changes, and ``remove`` deletes it. A
+prefix that holds something without a stamp is the user's own environment: ``verify``
+says what it lacks of what the recipe asks for, and nothing ever changes or deletes it.
+
+TOML layout (every section but ``[tool]`` and ``[locate]`` is optional)::
+
+    [tool]
+    name = "openbabel"
+    label = "OpenBabel"
+    version = "3.1.1"                       # pinned here; never a user option
+    prefix = "{tools_home}/{name}/{version}"  # the default
+
+    [locate]
+    paths = ["{prefix}/bin/obabel"]         # where the install puts it
+    commands = ["obabel"]                   # looked up on PATH
+    check = ["{exe}", "-V"]                 # {exe} = the candidate
+
+    [env]                                   # a conda prefix at {prefix}
+    conda = ["openbabel={version}"]
+    pip = []
+
+    [[install]]                             # plain steps, after [env]
+    name = "download the model"
+    argv = ["curl", "-fsSL", "-o", "{prefix}/model.pt", "https://..."]
+    timeout = 1800
+
+Every command runs through ``sh`` on the host, so a host is a POSIX machine.
+"""
+from __future__ import annotations
+
+import dataclasses
+import hashlib
+import io
+import json
+import re
+import shlex
+import subprocess
+import tarfile
+import threading
+import tomllib
+from concurrent.futures import Future
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterable
+
+from ms_env.steps import Step, StepFailed, run_step
+from ms_env.host import LOCAL, Host
+
+READY = "ready"
+MISSING = "missing"
+UNREACHABLE = "unreachable"
+_SSH_CONNECT_FAILED = 255  # ssh's own exit code; anything else came from the remote command
+STAMP = ".ms_env.json"  # inside a prefix ms_env installed
+_INSTALLING = ".installing"  # beside a prefix while it is installed: what a cut install leaves is ours too
+_USER = "user"
+
+
+# ----------------------------------------------------------------------------
+# Spec (the TOML, still templated) -> Recipe (concrete for one host)
+# ----------------------------------------------------------------------------
+
+@dataclass(frozen=True, slots=True)
+class RecipeStep:
+    name: str
+    argv: tuple[str, ...]
+    timeout: float = 1800.0
+    # Files of this machine the command reads as a tar, each under its own name. Out of the
+    # repr, which is what the stamp of a tool is made of.
+    send: tuple[str, ...] = field(default=(), repr=False)
+
+
+@dataclass(frozen=True, slots=True)
+class Recipe:
+    name: str
+    prefix: str
+    candidates: tuple[str, ...]  # executables tried first, in order
+    commands: tuple[str, ...]  # then these names on PATH
+    check: tuple[str, ...]  # "{exe}" marks the candidate
+    steps: tuple[RecipeStep, ...]
+    conda: tuple[str, ...] = ()  # what the environment at `prefix` has to hold
+    pip: tuple[str, ...] = ()
+    stamp: str = ""  # hash of what the TOML asks for, the same on every host
+    # What brings a prefix up to this recipe without making it again, for one of ours whose
+    # stamp names the same `base`. Without a base a recipe that changed is installed from nothing.
+    refresh: tuple[RecipeStep, ...] = ()
+    base: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class ToolSpec:
+    name: str
+    version: str
+    info: dict[str, Any]  # the whole [tool] table: label, purpose, footprint...
+    prefix: str
+    paths: tuple[str, ...]
+    commands: tuple[str, ...]
+    check: tuple[str, ...]
+    conda: tuple[str, ...] = ()
+    pip: tuple[str, ...] = ()
+    channels: tuple[str, ...] = ("conda-forge",)
+    steps: tuple[RecipeStep, ...] = ()
+
+    @classmethod
+    def load(cls, path: Path | str) -> "ToolSpec":
+        data = tomllib.loads(Path(path).read_text(encoding="utf-8"))
+        tool, locate, env = data["tool"], data["locate"], data.get("env", {})
+        return cls(
+            name=str(tool["name"]),
+            version=str(tool.get("version", "")),
+            info=dict(tool),
+            prefix=str(tool.get("prefix", "{tools_home}/{name}/{version}")),
+            paths=tuple(locate.get("paths", ())),
+            commands=tuple(locate.get("commands", ())),
+            check=tuple(locate.get("check", ())),
+            conda=tuple(env.get("conda", ())),
+            pip=tuple(env.get("pip", ())),
+            channels=tuple(env.get("channels", ("conda-forge",))),
+            steps=tuple(
+                RecipeStep(str(step["name"]), tuple(step["argv"]), float(step.get("timeout", 1800)))
+                for step in data.get("install", ())
+            ),
+        )
+
+    def recipe(
+        self,
+        tools_home: str,
+        *,
+        prefix: str = "",
+        extra_paths: Iterable[str] = (),
+        manager: str = "",
+    ) -> Recipe:
+        """Bind the templates to one host. ``tools_home``/``prefix`` are paths *on that host*.
+
+        ``prefix`` is a location the user chose instead of the managed one: an environment of
+        their own (checked, never changed) or a place to install into. ``extra_paths`` are
+        user-registered executables; ``manager`` is a conda-type program to use there instead
+        of micromamba.
+        """
+        values = {"tools_home": str(tools_home).rstrip("/"), "name": self.name, "version": self.version, "exe": "{exe}"}
+        values["prefix"] = str(prefix or self.prefix.format_map(values)).rstrip("/")
+        fill = lambda items: tuple(str(item).format_map(values) for item in items)  # noqa: E731
+        steps = _env_steps(self.name, fill(self.conda), fill(self.pip), fill(self.channels), values, manager)
+        steps += [RecipeStep(step.name, fill(step.argv), step.timeout) for step in self.steps]
+        return Recipe(
+            name=self.name,
+            prefix=values["prefix"],
+            candidates=(*map(str, extra_paths), *fill(self.paths)),
+            commands=self.commands,
+            check=fill(self.check),
+            steps=tuple(steps),
+            conda=fill(self.conda),
+            pip=fill(self.pip),
+            stamp=hashlib.sha256(
+                repr((self.name, self.version, self.conda, self.pip, self.channels, self.steps)).encode()
+            ).hexdigest()[:16],
+        )
+
+
+def _env_steps(
+    name: str, conda: tuple[str, ...], pip: tuple[str, ...], channels: tuple[str, ...], values: dict, manager: str
+) -> list[RecipeStep]:
+    if not conda and not pip:
+        return []
+    prefix = values["prefix"]
+    channel_args = [part for channel in channels for part in ("--channel", channel)]
+    create = shlex.join(["create", "--yes", "--quiet", "--prefix", prefix, *channel_args, *conda])
+    steps = [
+        RecipeStep(
+            f"create the {name} environment",
+            ("sh", "-c", f'{_find_or_bootstrap_manager(manager, values["tools_home"])}\n"$M" {create}'),
+        )
+    ]
+    if pip:
+        steps.append(
+            RecipeStep(
+                f"pip install into {name}",
+                (f"{prefix}/bin/python", "-m", "pip", "install", "--progress-bar", "off", *pip),
+                timeout=3600.0,
+            )
+        )
+    return steps
+
+
+def _find_or_bootstrap_manager(hint: str, tools_home: str) -> str:
+    """sh that leaves the manager in $M: the hint, micromamba on PATH, or one it downloads."""
+    root = shlex.quote(f"{tools_home}/micromamba")
+    return f"""M={shlex.quote(str(hint or ""))}
+[ -x "$M" ] || M=$(command -v micromamba) || {{
+  case "$(uname -s)-$(uname -m)" in
+    Linux-x86_64) p=linux-64 ;; Linux-aarch64) p=linux-aarch64 ;;
+    Darwin-arm64) p=osx-arm64 ;; Darwin-x86_64) p=osx-64 ;;
+    *) echo "no micromamba here and no build of it for $(uname -sm)" >&2; exit 1 ;;
+  esac
+  M={root}/bin/micromamba
+  if [ ! -x "$M" ]; then
+    echo "no micromamba found: installing it into {root}"
+    mkdir -p {root} && curl -fsSL "https://micro.mamba.pm/api/micromamba/$p/latest" | tar -xj -C {root} bin/micromamba || exit 1
+  fi
+  export MAMBA_ROOT_PREFIX={root}
+}}"""
+
+
+# ----------------------------------------------------------------------------
+# A recipe on a host
+# ----------------------------------------------------------------------------
+
+def _argv_on(argv: Iterable[str], host: Host) -> list[str]:
+    return list(argv) if host.address is None else host.argv(shlex.join(argv))
+
+
+def _prefix(recipe: Recipe) -> str:
+    if not recipe.prefix.strip("/ "):
+        raise ValueError(f"{recipe.name}: a recipe needs a prefix of its own, not {recipe.prefix!r}.")
+    return shlex.quote(recipe.prefix)
+
+
+def _stamp_text(recipe: Recipe) -> str:
+    return json.dumps({"name": recipe.name, "stamp": recipe.stamp, **({"base": recipe.base} if recipe.base else {})})
+
+
+def _base(owner: str) -> str:
+    """The base a stamp names; "" for what is not a stamp, or the stamp of a recipe without one."""
+    try:
+        return json.loads(owner).get("base", "")
+    except (ValueError, AttributeError):
+        return ""
+
+
+def _tar(paths: Iterable[str]) -> bytes:
+    held = io.BytesIO()
+    with tarfile.open(fileobj=held, mode="w") as tar:
+        for path in paths:
+            tar.add(path, arcname=Path(path).name)
+    return held.getvalue()
+
+
+def _step(step: RecipeStep, label: str, host: Host) -> Step:
+    if not step.send:
+        return Step(label + step.name, argv=_argv_on(step.argv, host), timeout=step.timeout)
+
+    def send() -> str:
+        host.run(shlex.join(step.argv), data=_tar(step.send), timeout=step.timeout)
+        return "sent " + ", ".join(Path(path).name for path in step.send)
+
+    return Step(label + step.name, call=send, timeout=step.timeout)
+
+
+def recipe_steps(recipe: Recipe, host: Host = LOCAL, refresh: bool = False) -> list[Step]:
+    """The whole install: clear what ms_env left at the prefix, run the recipe, stamp it.
+
+    The first step fails on a prefix that holds something ms_env did not install. With
+    `refresh` the prefix stays and only `recipe.refresh` runs before the stamp.
+    """
+    label, prefix = (f"{host.address}: " if host.address else ""), _prefix(recipe)
+    clear = (
+        f'p={prefix}\n'
+        f'if [ -n "$(ls -A "$p" 2>/dev/null)" ] && [ ! -f "$p/{STAMP}" ] && [ ! -f "$p{_INSTALLING}" ]; then\n'
+        '  echo "$p holds an environment ms_env did not install: it is checked, never changed." >&2; exit 1\n'
+        "fi\n"
+        f'mkdir -p "$(dirname "$p")" && : > "$p{_INSTALLING}" && rm -rf -- "$p"'
+    )
+    stamp = (
+        f'p={prefix}\nmkdir -p "$p" && printf %s {shlex.quote(_stamp_text(recipe))} > "$p/{STAMP}" '
+        f'&& rm -f "$p{_INSTALLING}"'
+    )
+    steps = (
+        *(recipe.refresh if refresh else (RecipeStep(f"prepare {recipe.prefix}", ("sh", "-c", clear), 300.0), *recipe.steps)),
+        RecipeStep(f"record the {recipe.name} install", ("sh", "-c", stamp), 60.0),
+    )
+    return [_step(step, label, host) for step in steps]
+
+
+def _locate_script(recipe: Recipe) -> str:
+    check = " ".join('"$1"' if part == "{exe}" else shlex.quote(part) for part in recipe.check) or "true"
+    return (
+        f'ok() {{ [ -x "$1" ] && {check} >/dev/null 2>&1; }}\n'
+        f"for p in {' '.join(map(shlex.quote, recipe.candidates))}; do"
+        ' ok "$p" && { echo "$p"; exit 0; }; done\n'
+        f"for c in {' '.join(map(shlex.quote, recipe.commands))}; do"
+        ' p=$(command -v "$c") && ok "$p" && { echo "$p"; exit 0; }; done\n'
+        "exit 1"
+    )
+
+
+def locate(recipe: Recipe, host: Host = LOCAL, timeout: float = 60.0) -> tuple[str, str]:
+    """(READY, executable) or (MISSING | UNREACHABLE, ""). Blocks on a subprocess: keep it off UI threads."""
+    try:
+        result = subprocess.run(
+            host.argv(_locate_script(recipe)),
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        return (UNREACHABLE if host.address else MISSING), ""
+    found = result.stdout.strip().splitlines()
+    if result.returncode == 0 and found:
+        return READY, found[-1]
+    return (UNREACHABLE if host.address and result.returncode == _SSH_CONNECT_FAILED else MISSING), ""
+
+
+def status(recipe: Recipe, host: Host = LOCAL) -> str:
+    """READY, MISSING or UNREACHABLE. Blocks like `locate`, which also says where the tool is."""
+    return locate(recipe, host)[0]
+
+
+# ----------------------------------------------------------------------------
+# Whose the prefix is, and whether it holds what the recipe asks for
+# ----------------------------------------------------------------------------
+
+def _owner(recipe: Recipe, host: Host) -> str:
+    """The stamp of a prefix ms_env installed, `_USER` for one that holds something else, "" for none."""
+    script = (
+        f'p={_prefix(recipe)}\n'
+        f'[ -f "$p/{STAMP}" ] && {{ cat "$p/{STAMP}"; exit 0; }}\n'
+        f'[ -f "$p{_INSTALLING}" ] && {{ echo installing; exit 0; }}\n'
+        f'[ -n "$(ls -A "$p" 2>/dev/null)" ] && echo {_USER}\nexit 0'
+    )
+    return host.run(script, timeout=60).decode().strip()
+
+
+def _pin(spec: str) -> tuple[str, str]:
+    """(name, version) of `name=1.2`, `name==1.2` or `name[extra]==1.2`. A range pins no version."""
+    name, version = re.match(r"\s*([\w.-]*)(?:\[[^]]*])?\s*(?:==?\s*([\w.!+-]+))?", spec).groups()
+    return name, (version or "").rstrip(".")
+
+
+# Run by the python of the prefix. ponytail: versions compare as text, so 1.0 is not 1.0.0;
+# `packaging` on this side if a recipe ever pins that loosely.
+_PIP_PINS = """import sys
+from importlib.metadata import PackageNotFoundError, version
+for pin in sys.argv[1:]:
+    name, _, wanted = pin.partition("==")
+    try:
+        found = version(name)
+    except PackageNotFoundError:
+        found = "nothing"
+    if found == "nothing" or (wanted and found != wanted):
+        print(f"pip: {pin} is asked for, {found} is there")
+"""
+
+
+def verify(recipe: Recipe, host: Host = LOCAL) -> list[str]:
+    """What the environment at the recipe's prefix lacks of what the recipe asks for; [] when it meets it.
+
+    The way to take an environment the user already has instead of installing one: every
+    pinned conda package (looked up in `conda-meta`), every pinned pip package (asked to the
+    python of the prefix) and the recipe's check on the executables of that prefix. Reads
+    only. ponytail: a pip URL or path is not looked up, and `>=` counts as "is there".
+    """
+    lines = [f'p={_prefix(recipe)}', '[ -d "$p" ] || { echo "there is no directory $p"; exit 0; }',
+             'has() { ls "$p"/conda-meta/$1 >/dev/null 2>&1; }']
+    for name, version in map(_pin, recipe.conda):
+        # conda's `=1.2` takes 1.2 and 1.2.x; a file there is name-version-build.json
+        globs = [f"{name}-{version}-*.json", f"{name}-{version}.*-*.json"] if version else [f"{name}-[0-9]*-*.json"]
+        wanted = shlex.quote(f"conda: {name}{'=' + version if version else ''} is asked for and is not there")
+        lines.append(" || ".join([*(f"has {shlex.quote(glob)}" for glob in globs), f"echo {wanted}"]))
+    pins = ["==".join(filter(None, _pin(spec))) for spec in recipe.pip if "/" not in spec]
+    if pins:
+        lines.append(
+            f'"$p/bin/python" -c {shlex.quote(_PIP_PINS)} {shlex.join(pins)} 2>/dev/null'
+            ' || echo "pip: no working python at $p/bin/python"'
+        )
+    lacks = host.run("\n".join(lines), timeout=120).decode().splitlines()
+    inside = tuple(path for path in recipe.candidates if path.startswith(recipe.prefix + "/"))
+    if recipe.check and status(dataclasses.replace(recipe, candidates=inside, commands=()), host) != READY:
+        lacks.append(f"check: `{shlex.join(recipe.check)}` passes for none of {', '.join(inside) or 'its executables'}")
+    return lacks
+
+
+def remove(recipe: Recipe, host: Host = LOCAL) -> bool:
+    """Delete the prefix when ms_env installed it. False: nothing of ours there, nothing deleted."""
+    key = f"{recipe.name}@{host.address or 'local'}"
+    with _LOCK:
+        current = _INSTALLS.get(key)
+        if current is not None and not current.future.done():
+            raise RuntimeError(f"{recipe.name} is being installed on {host.address or 'this machine'}.")
+    script = (
+        f'p={_prefix(recipe)}\n[ -f "$p/{STAMP}" ] || [ -f "$p{_INSTALLING}" ] || {{ echo no; exit 0; }}\n'
+        f'rm -rf -- "$p" "$p{_INSTALLING}"'
+    )
+    return host.run(script, timeout=600).decode().strip() != "no"
+
+
+# ----------------------------------------------------------------------------
+# Installs: app-level background threads
+# ----------------------------------------------------------------------------
+
+@dataclass
+class Install:
+    key: str
+    future: Future = field(default_factory=Future)
+    lines: list[str] = field(default_factory=list)  # live log, appended from the install thread
+
+
+_INSTALLS: dict[str, Install] = {}
+_LOCK = threading.Lock()
+
+
+def ensure(recipe: Recipe, host: Host = LOCAL, busy: Callable[[], str] | None = None) -> Install:
+    """Make sure the tool is there: start installing it, or join the install already running.
+
+    Returns at once: wait on ``future`` and read ``lines`` for the log. Nothing is installed
+    where the tool is already usable. An install of ours made from another recipe is made
+    again, or brought up to this one where it stands when the recipe says how (that is the
+    update); an environment of the user is verified and left as it is.
+
+    ``busy`` is asked only before an install of ours is changed: what it returns is why that
+    cannot be done now (something is running from it), and the install fails saying so.
+    """
+    key = f"{recipe.name}@{host.address or 'local'}"
+    with _LOCK:
+        current = _INSTALLS.get(key)
+        if current is not None and not current.future.done():
+            return current
+        install = _INSTALLS[key] = Install(key)
+    # ponytail: one daemon thread per install; a pool if installs pile up.
+    # Daemon on purpose: quitting the app must not wait 30 min for a conda solve.
+    threading.Thread(target=_run_install, args=(install, recipe, host, busy), name=f"ms_env-{key}", daemon=True).start()
+    return install
+
+
+def _run_install(install: Install, recipe: Recipe, host: Host, busy: Callable[[], str] | None = None) -> None:
+    install.future.set_running_or_notify_cancel()
+    try:
+        where = host.address or "this machine"
+        owner = _owner(recipe, host)
+        if owner == _USER:
+            lacks = verify(recipe, host)
+            if lacks:
+                raise StepFailed(
+                    f"{recipe.prefix} on {where} does not meet the {recipe.name} recipe:\n" + "\n".join(lacks)
+                )
+            install.lines.append(f"{recipe.prefix} on {where} meets the {recipe.name} recipe: used as it is.")
+        elif owner in ("", _stamp_text(recipe)) and status(recipe, host) == READY:
+            install.lines.append(f"{recipe.name} is already usable on {where}.")
+        else:
+            why = busy() if busy and owner.startswith("{") else ""  # a stamp: an install of ours that ended
+            if why:
+                raise StepFailed(f"{recipe.name} on {where} is not as this recipe asks, and is left as it is: {why}")
+            for step in recipe_steps(recipe, host, refresh=bool(recipe.base) and _base(owner) == recipe.base):
+                run_step(step, install.lines.append)
+            state = status(recipe, host)
+            if state != READY:
+                raise StepFailed(f"{recipe.name}: every step ran but the tool is still {state} on {where}.")
+    except Exception as exc:  # noqa: BLE001 -- whatever broke, dependents must see a failure
+        install.lines.append(str(exc))
+        install.future.set_exception(exc)
+    else:
+        install.future.set_result(None)
+
+
+__all__ = [
+    "Install",
+    "MISSING",
+    "READY",
+    "Recipe",
+    "RecipeStep",
+    "STAMP",
+    "ToolSpec",
+    "UNREACHABLE",
+    "ensure",
+    "locate",
+    "recipe_steps",
+    "remove",
+    "status",
+    "verify",
+]

ms_env/steps.py

@@ -0,0 +1,94 @@
+"""Steps: named units of work that run one after another, their output read line by line.
+
+Plain standard library and Qt-free: a UI only renders `Step` lists and the lines they print.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Iterable, Optional
+
+DEFAULT_TIMEOUT = 30.0
+
+
+class StepFailed(RuntimeError):
+    pass
+
+
+@dataclass(slots=True)
+class Step:
+    """One named unit of work: either a command line or an in-process call."""
+
+    name: str
+    argv: Optional[list[str]] = None
+    call: Optional[Callable[[], str]] = None
+    timeout: float = DEFAULT_TIMEOUT
+
+
+def run_step(step: Step, on_line: Callable[[str], None] | None = None) -> None:
+    """Run one step, streaming its output. Raises StepFailed on failure."""
+    emit = on_line or (lambda _line: None)
+    if step.call is not None:
+        message = step.call()
+        if message:
+            emit(message)
+        return
+    if not step.argv:
+        return
+    emit("$ " + " ".join(step.argv))
+    try:
+        process = subprocess.Popen(
+            step.argv,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+        )
+    except FileNotFoundError as exc:
+        raise StepFailed(f"{step.name}: command not found ({exc.filename}).") from exc
+    deadline = time.monotonic() + step.timeout
+    assert process.stdout is not None
+    for line in process.stdout:
+        emit(line.rstrip())
+        if time.monotonic() > deadline:
+            process.kill()
+            raise StepFailed(f"{step.name}: timed out after {step.timeout:.0f}s.")
+    try:
+        code = process.wait(timeout=max(1.0, deadline - time.monotonic()))
+    except subprocess.TimeoutExpired:
+        process.kill()
+        raise StepFailed(f"{step.name}: timed out after {step.timeout:.0f}s.") from None
+    if code != 0:
+        raise StepFailed(f"{step.name}: exited with code {code}.")
+
+
+def run_steps(
+    steps: Iterable[Step],
+    on_line: Callable[[str], None] | None = None,
+    on_step: Callable[[int, Step], None] | None = None,
+) -> None:
+    for index, step in enumerate(steps):
+        if on_step is not None:
+            on_step(index, step)
+        run_step(step, on_line)
+
+
+def find_environment_manager(*extra_variables: str) -> Path | None:
+    """The program that creates environments here: micromamba, unless a variable names another.
+
+    ``extra_variables`` are app-specific overrides, read before ``MOLSUITE_ENV_MANAGER``; any
+    of them may name a conda or a mamba. None: the recipe downloads its own micromamba.
+    """
+    named = [str(os.environ.get(variable) or "").strip() for variable in (*extra_variables, "MOLSUITE_ENV_MANAGER")]
+    for candidate in (*filter(None, named), shutil.which("micromamba")):
+        path = Path(candidate or "").expanduser()
+        if path.is_file() and os.access(path, os.X_OK):
+            return path.resolve()
+    return None
+
+
+__all__ = ["DEFAULT_TIMEOUT", "Step", "StepFailed", "find_environment_manager", "run_step", "run_steps"]

ms_env-0.2.0.dist-info/METADATA

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: ms_env
+Version: 0.2.0
+Summary: Environments for MolSuite apps: find or install external tools, on this machine or over ssh.
+Author: Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/MolSuite/ms_env
+Project-URL: Documentation, https://molsuite.github.io/ms_env/
+Project-URL: Issues, https://github.com/MolSuite/ms_env/issues
+Keywords: environments,conda,micromamba,ssh,molsuite
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Dynamic: license-file
+
+# ms_env
+
+Environments for MolSuite apps: find an external tool that already works, or install
+it, on this machine or on a host reached over `ssh`. Standard library only.
+
+A **recipe** says how to tell that a tool works and how to install it. A **host**
+says where. `ms_env` applies one to the other.
+
+```python
+from ms_env import READY, Host, ToolSpec, ensure, locate
+
+spec = ToolSpec.load("openbabel.toml")
+
+recipe = spec.recipe("/home/me/.myapp/tools")      # bound to this machine
+state, executable = locate(recipe)                  # (READY, ".../bin/obabel") or (MISSING, "")
+if state != READY:
+    install = ensure(recipe)                        # returns at once: a daemon thread installs
+    install.future.result()                         # wait; `install.lines` is the live log
+
+cluster = Host("me@login.hpc")                    # as `ssh` knows it
+ensure(spec.recipe("/home/me/.myapp/tools"), cluster).future.result()
+```
+
+## A recipe
+
+One TOML file per tool, shipped by the app:
+
+```toml
+[tool]
+name = "openbabel"
+label = "OpenBabel"
+version = "3.1.1"                         # pinned here; never a user option
+prefix = "{tools_home}/{name}/{version}"  # the default
+
+[locate]
+paths = ["{prefix}/bin/obabel"]           # where the install puts it
+commands = ["obabel"]                     # looked up on PATH
+check = ["{exe}", "-V"]                   # {exe} = the candidate
+
+[env]                                     # a conda prefix at {prefix}
+conda = ["openbabel={version}"]
+pip = []
+
+[[install]]                               # plain steps, after [env]
+name = "download the model"
+argv = ["curl", "-fsSL", "-o", "{prefix}/model.pt", "https://..."]
+timeout = 1800
+```
+
+Lookup on a host, first candidate whose check exits 0 wins: the paths the user
+registered, the managed install under `tools_home`, then `PATH`. `[env]` is created
+with micromamba: the one on the host's `PATH`, or one downloaded into `tools_home`.
+Another conda-type program is used only when the app names it (`manager=`).
+
+## Whose an environment is
+
+`ensure` stamps the prefix it installs. The stamp decides what may happen to it:
+
+| At the prefix | `ensure` | `remove` |
+|---|---|---|
+| Nothing | Creates it | Nothing to delete |
+| A stamp of this recipe | Nothing | Deletes it |
+| A stamp of another recipe (the TOML changed) | Creates it again: the update | Deletes it |
+| Something without a stamp: the user's own environment | Verifies it, never changes it | Leaves it |
+
+```python
+from ms_env import remove, verify
+
+mine = spec.recipe("/home/me/.myapp/tools", prefix="/home/me/envs/babel")   # an environment I already have
+verify(mine)    # [] when it meets the recipe, else what it lacks:
+                # ["conda: openbabel=3.1.1 is asked for and is not there"]
+remove(recipe)  # True: ms_env had installed it and it is gone
+```
+
+`verify` looks up every pinned conda package, every pinned pip package and the
+recipe's check, and only reads. An executable registered on its own (`extra_paths=`)
+is not an environment: it only has to pass the check.
+
+`ensure(recipe, host, busy=why)` asks `why()` before it changes an install of its own:
+a reason (`"2 jobs are running from it"`) makes the install fail and leaves the prefix
+as it is.
+
+## A host
+
+`Host()` is this machine. `Host("me@login.hpc")` runs every command through the system's
+`ssh` with `BatchMode=yes`: users, keys and jumps come from `~/.ssh/config`, the key
+logs in or the command fails at once, no credential is handled and nothing is left
+listening on the other side. A host is a POSIX machine; only Linux is supported.
+
+## The environment of an app
+
+`app_recipe` makes the recipe that gives a host the environment an app runs in here:
+the same Python, the same versions, and the code of a source checkout as it is now.
+
+```python
+from ms_env import app_recipe
+
+home = cluster.home()
+recipe = app_recipe("myapp", ["myapp"], ["myapp"], prefix=f"{home}/.myapp/env",
+                    tools_home=f"{home}/.myapp/tools", cache="/home/me/.myapp/cache/wheels")
+ensure(recipe, cluster).future.result()
+```
+
+## Install
+
+```bash
+pip install ms_env
+```
+
+Documentation: <https://molsuite.github.io/ms_env/>

ms_env-0.2.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+ms_env/__init__.py,sha256=RzjgrvC-z3s6rlaiG8A7ilt4hsZpIT66kXuj6Yv-_zA,1019
+ms_env/app.py,sha256=DSD-Z5hPmARL_hi7_VYhczCDb_0RZCJIyEcOlq2y_9U,7457
+ms_env/host.py,sha256=1WA4vibBeWo1J3T9oJzpxOAK1u96uY_N7kua57mcJb4,2316
+ms_env/recipes.py,sha256=0cusgSc4HFKxDhZEZgWZgH090Qdascy9cyd6u7fMlVg,20521
+ms_env/steps.py,sha256=mM-W3bAjrB-0URHT8cwt0vk5TPfmWSUrEojJj40RodY,3143
+ms_env-0.2.0.dist-info/licenses/LICENSE,sha256=BnhLi3mlB9CxUBdj6SKp9izP481uYTrbbWvWii7ZrLE,1065
+ms_env-0.2.0.dist-info/METADATA,sha256=muPayPCnvP8qVbPrbejfYdbk4D9rOhGEIj8cd4o7fi0,5096
+ms_env-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+ms_env-0.2.0.dist-info/top_level.txt,sha256=DzhK1x5Ja_vgzaa0wEhRpY7UrBL8eY6zxfLzbFtANh8,7
+ms_env-0.2.0.dist-info/RECORD,,

ms_env-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ms_env-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 MolSuite
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ms_env-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+ms_env