everyome 0.1.0a1__py3-none-any.whl

everyome/__init__.py

@@ -0,0 +1,2 @@
+"""Everyome: every -ome, for everyone. Created by Helen R Horkan PhD (Stowers Institute); please cite it if you use it."""
+__version__ = "0.1.0a1"

everyome/__main__.py

@@ -0,0 +1,30 @@
+"""Start Everyome: `everyome` (or `python -m everyome`) opens the app in your browser."""
+import argparse
+import os
+import threading
+import webbrowser
+
+import uvicorn
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Everyome: every -ome, for everyone")
+    ap.add_argument("--port", type=int, default=8777)
+    ap.add_argument("--no-browser", action="store_true", help="do not open a browser window")
+    a = ap.parse_args()
+    try:
+        import setproctitle
+        # Intent: Stowers policy names every process Claude_<user>_<tool>_<stage>.
+        setproctitle.setproctitle(f"Claude_{os.environ.get('USER', 'user')}_everyome_app")
+    except ImportError:
+        pass
+    url = f"http://127.0.0.1:{a.port}"
+    if not a.no_browser:
+        threading.Timer(1.2, lambda: webbrowser.open(url)).start()
+    print(f"Everyome is running at {url}  (press Ctrl+C to quit)")
+    # Intent: listen on localhost only; the app is for this computer's user.
+    uvicorn.run("everyome.core.server:app", host="127.0.0.1", port=a.port, log_level="warning")
+
+
+if __name__ == "__main__":
+    main()

everyome/core/__init__.py

@@ -0,0 +1 @@
+"""Core: backends, scheduler, projects, runs and the module loader (see docs/PLAN.md, section 3)."""

everyome/core/backends.py

@@ -0,0 +1,166 @@
+"""How the app talks to the computer that does the work (carried over from SPLiTscape).
+
+Two backends with one interface:
+  * SSHBackend   — a real cluster. Uses the system `ssh` client, so the user's
+                   key, ssh-agent, ~/.ssh/config and jump hosts all apply. The
+                   app never reads the private key. BatchMode means it never
+                   prompts for a password: key-based login only.
+  * LocalBackend — this computer, inside a folder. With fake Slurm on PATH it is
+                   the demo cluster; without it, it runs local modules.
+
+Both expose: run, read_bytes, write_bytes, listdir, exists, tunnel, close.
+"""
+from __future__ import annotations
+
+import os
+import shlex
+import socket
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class Entry:
+    name: str
+    is_dir: bool
+    size: int
+
+
+def free_port() -> int:
+    with socket.socket() as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+class Backend:
+    label = "cluster"
+    user = "unknown"
+
+    def run(self, cmd: str, timeout: float = 60, stdin: bytes | None = None) -> tuple[int, str, str]:
+        raise NotImplementedError
+
+    def read_bytes(self, path: str) -> bytes:
+        rc, out, err = self.run_bytes(f"cat {shlex.quote(path)}")
+        if rc:
+            raise FileNotFoundError(f"{path}: {err.strip()}")
+        return out
+
+    def read_text(self, path: str) -> str:
+        return self.read_bytes(path).decode()
+
+    def write_bytes(self, path: str, data: bytes) -> None:
+        q = shlex.quote(path)
+        rc, _, err = self.run(f"mkdir -p $(dirname {q}) && cat > {q}", stdin=data)
+        if rc:
+            raise OSError(f"could not write {path}: {err.strip()}")
+
+    def write_text(self, path: str, text: str) -> None:
+        self.write_bytes(path, text.encode())
+
+    def exists(self, path: str) -> bool:
+        return self.run(f"test -e {shlex.quote(path)}")[0] == 0
+
+    def listdir(self, path: str) -> list[Entry]:
+        # Intent: portable listing (type, size, name), one entry per line.
+        cmd = (f"cd {shlex.quote(path)} && for f in * .[!.]*; do [ -e \"$f\" ] || continue; "
+               "if [ -d \"$f\" ]; then t=d; s=0; else t=f; s=$(wc -c < \"$f\" | tr -d ' '); fi; "
+               "printf '%s\\t%s\\t%s\\n' \"$t\" \"$s\" \"$f\"; done")
+        rc, out, err = self.run(cmd)
+        if rc:
+            raise FileNotFoundError(f"{path}: {err.strip() or 'not a folder'}")
+        rows = [line.split("\t", 2) for line in out.splitlines() if line.count("\t") >= 2]
+        return sorted((Entry(n, t == "d", int(s)) for t, s, n in rows), key=lambda e: (not e.is_dir, e.name.lower()))
+
+    def run_bytes(self, cmd: str, timeout: float = 120) -> tuple[int, bytes, str]:
+        raise NotImplementedError
+
+    def tunnel(self, node: str, remote_port: int) -> int:
+        """Forward a local port to node:remote_port; returns the local port."""
+        raise NotImplementedError
+
+    def close(self) -> None:
+        pass
+
+
+class LocalBackend(Backend):
+    """This computer: a working folder, optionally with fake Slurm commands (the demo cluster)."""
+
+    def __init__(self, root: Path, fake_slurm_bin: Path | None = None, label: str = "this computer"):
+        self.root = root
+        self.label = label
+        self.env = dict(os.environ)
+        if fake_slurm_bin:
+            self.env["PATH"] = f"{fake_slurm_bin}{os.pathsep}{self.env.get('PATH', '')}"
+            self.env["FAKE_SLURM_HOME"] = str(root / ".slurm")
+        # Intent: jobs import this same copy of Everyome, as they would from a cluster install.
+        self.env["PYTHONPATH"] = os.pathsep.join(filter(None, [str(Path(__file__).resolve().parents[2]), self.env.get("PYTHONPATH")]))
+        self.user = os.environ.get("USER", "user")
+        self.host = "localhost"
+
+    def run(self, cmd, timeout=60, stdin=None):
+        p = subprocess.run(["bash", "-c", cmd], cwd=self.root, env=self.env, input=stdin,
+                           capture_output=True, timeout=timeout)
+        return p.returncode, p.stdout.decode(errors="replace"), p.stderr.decode(errors="replace")
+
+    def run_bytes(self, cmd, timeout=120):
+        p = subprocess.run(["bash", "-c", cmd], cwd=self.root, env=self.env, capture_output=True, timeout=timeout)
+        return p.returncode, p.stdout, p.stderr.decode(errors="replace")
+
+    def tunnel(self, node, remote_port):
+        return remote_port          # same machine: nothing to forward
+
+
+class SSHBackend(Backend):
+    """Real cluster over OpenSSH with connection sharing (one login, many commands)."""
+
+    def __init__(self, host: str, user: str, port: int = 22, ssh_options: list[str] | None = None):
+        self.host, self.user, self.port = host, user, port
+        self.label = f"{user}@{host}"
+        ctl = Path.home() / ".ssh" / "everyome-%r@%h-%p"
+        (Path.home() / ".ssh").mkdir(mode=0o700, exist_ok=True)
+        # Intent: share one login across commands (not supported by Windows' OpenSSH).
+        share = [] if os.name == "nt" else ["-o", "ControlMaster=auto", "-o", f"ControlPath={ctl}", "-o", "ControlPersist=15m"]
+        self.base = ["ssh", "-p", str(port), "-o", "BatchMode=yes", "-o", "ConnectTimeout=15", *share,
+                     "-o", "ServerAliveInterval=30", *(ssh_options or []), f"{user}@{host}"]
+        self.tunnels: list[subprocess.Popen] = []
+
+    def _ssh(self, cmd, stdin=None, timeout=60):
+        # Intent: run under a login shell so the user's cluster PATH (Slurm, modules) is set.
+        return subprocess.run([*self.base, f"bash -lc {shlex.quote(cmd)}"], input=stdin, capture_output=True, timeout=timeout)
+
+    def run(self, cmd, timeout=60, stdin=None):
+        p = self._ssh(cmd, stdin, timeout)
+        return p.returncode, p.stdout.decode(errors="replace"), p.stderr.decode(errors="replace")
+
+    def run_bytes(self, cmd, timeout=120):
+        p = self._ssh(cmd, None, timeout)
+        return p.returncode, p.stdout, p.stderr.decode(errors="replace")
+
+    def tunnel(self, node, remote_port):
+        lport = free_port()
+        args = [a for a in self.base if a != f"{self.user}@{self.host}"]
+        proc = subprocess.Popen([*args, "-N", "-o", "ExitOnForwardFailure=yes",
+                                 "-L", f"127.0.0.1:{lport}:{node}:{remote_port}", f"{self.user}@{self.host}"],
+                                stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+        self.tunnels.append(proc)
+        return lport
+
+    def close(self):
+        for t in self.tunnels:
+            t.terminate()
+        subprocess.run([*self.base[:-1], "-O", "exit", f"{self.user}@{self.host}"], capture_output=True)
+
+
+def describe_ssh_error(err: str) -> str:
+    """Turn common ssh failures into plain-language advice."""
+    e = err.lower()
+    if "permission denied" in e:
+        return "The cluster refused your key. Check that your SSH key is added to the cluster and loaded (ssh-add)."
+    if "could not resolve" in e or "name or service not known" in e:
+        return "That cluster address could not be found. Check the host name, and that you are on the institute network or VPN."
+    if "timed out" in e or "no route" in e:
+        return "The cluster did not answer. Check you are on the institute network or VPN."
+    if "host key verification failed" in e:
+        return "Your computer has not seen this cluster before. Log in once from a terminal to accept its host key, or ask IT."
+    return err.strip() or "Connection failed."

everyome/core/demo/__init__.py

@@ -0,0 +1,38 @@
+"""The demo cluster: a folder on this computer plus a pretend Slurm.
+
+<root>/
+  projects/             where demo projects are saved
+  data/<module id>/     each module's small demo dataset
+  .bin/                 sbatch, squeue, sacct, scancel (fake_slurm.py)
+  .slurm/               the pretend scheduler's job records
+"""
+from __future__ import annotations
+
+import importlib
+import shlex
+import shutil
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+
+
+def setup(root: Path, modules: dict) -> Path:
+    """Create or refresh the demo cluster; returns the folder holding the fake Slurm commands."""
+    (root / "projects").mkdir(parents=True, exist_ok=True)
+    for m in modules.values():
+        if m.demo.get("data"):
+            # Intent: refresh demo data from the installed modules, never touching users' demo projects.
+            shutil.copytree(m.dir / m.demo["data"], root / "data" / m.id, dirs_exist_ok=True)
+        if m.demo.get("build") and not (root / "data" / m.id / ".demo_ready").exists():
+            # Intent: modules whose demo data is generated (too large to ship) build it once, here.
+            mod, _, fn = m.demo["build"].partition(":")
+            getattr(importlib.import_module(mod), fn)(root / "data" / m.id)
+            (root / "data" / m.id / ".demo_ready").write_text("1")
+    bin_dir = root / ".bin"
+    bin_dir.mkdir(exist_ok=True)
+    for cmd in ("sbatch", "squeue", "sacct", "scancel"):
+        f = bin_dir / cmd
+        f.write_text(f"#!/bin/bash\nexec {shlex.quote(sys.executable)} {shlex.quote(str(HERE / 'fake_slurm.py'))} {cmd} \"$@\"\n")
+        f.chmod(0o755)
+    return bin_dir

everyome/core/demo/fake_slurm.py

@@ -0,0 +1,151 @@
+"""A tiny stand-in for Slurm, used by the demo cluster.
+
+Implements the subset Everyome uses (carried over from SPLiTscape):
+  sbatch [--parsable] [--dependency=afterok:ID[:ID]] [--job-name=..] [--output=..] script
+  squeue -h -j IDS -o "%i %T %N"
+  sacct  -n -P -j IDS -o JobID,State,Elapsed
+  scancel ID [ID ...]
+
+Jobs run as background processes; state lives in $FAKE_SLURM_HOME.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import socket
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+HOME = Path(os.environ.get("FAKE_SLURM_HOME", Path.home() / ".fake_slurm"))
+JOBS = HOME / "jobs"
+
+
+def _load(jid):
+    return json.loads((JOBS / f"{jid}.json").read_text())
+
+
+def _save(job):
+    tmp = JOBS / f"{job['id']}.json.tmp"
+    tmp.write_text(json.dumps(job))
+    tmp.replace(JOBS / f"{job['id']}.json")
+
+
+def _directives(script: str) -> dict:
+    out = {}
+    for line in Path(script).read_text().splitlines():
+        m = re.match(r"#SBATCH\s+--([\w-]+)(?:[= ](.+))?", line.strip())
+        if m:
+            out[m.group(1)] = (m.group(2) or "").strip()
+    return out
+
+
+def sbatch(args):
+    JOBS.mkdir(parents=True, exist_ok=True)
+    opts, script = {}, None
+    for a in args:
+        if a.startswith("--"):
+            k, _, v = a[2:].partition("=")
+            opts[k] = v
+        else:
+            script = a
+    if not script:
+        sys.exit("sbatch: no script given")
+    opts = {**_directives(script), **opts}
+    counter = HOME / "counter"
+    jid = int(counter.read_text()) + 1 if counter.exists() else 1001
+    counter.write_text(str(jid))
+    deps = []
+    if opts.get("dependency", "").startswith("afterok:"):
+        deps = [int(x) for x in opts["dependency"][8:].split(":") if x]
+    out = opts.get("output", str(Path.cwd() / f"slurm-{jid}.out")).replace("%j", str(jid))
+    job = {"id": jid, "name": opts.get("job-name", Path(script).name), "script": str(Path(script).resolve()),
+           "cwd": str(Path.cwd()), "output": out, "deps": deps, "state": "PENDING", "start": None, "end": None,
+           "node": socket.gethostname(), "pid": None}
+    _save(job)
+    # Intent: detach a runner that waits for dependencies, then executes the script.
+    subprocess.Popen([sys.executable, __file__, "_run", str(jid)], start_new_session=True,
+                     stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=os.environ.copy())
+    print(jid if "parsable" in opts else f"Submitted batch job {jid}")
+
+
+def _run(jid):
+    job = _load(jid)
+    while True:
+        states = [_load(d)["state"] for d in job["deps"]]
+        if any(s in ("FAILED", "CANCELLED", "TIMEOUT") for s in states):
+            job = _load(jid); job.update(state="CANCELLED", reason="DependencyNeverSatisfied", end=time.time()); _save(job); return
+        if all(s == "COMPLETED" for s in states):
+            break
+        if _load(jid)["state"] == "CANCELLED":
+            return
+        time.sleep(0.5)
+    job = _load(jid)
+    if job["state"] == "CANCELLED":
+        return
+    Path(job["output"]).parent.mkdir(parents=True, exist_ok=True)
+    with open(job["output"], "w") as log:
+        env = dict(os.environ, SLURM_JOB_ID=str(jid), SLURMD_NODENAME=job["node"])
+        p = subprocess.Popen(["bash", job["script"]], cwd=job["cwd"], stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True)
+        job.update(state="RUNNING", start=time.time(), pid=p.pid); _save(job)
+        rc = p.wait()
+    job = _load(jid)
+    if job["state"] != "CANCELLED":
+        job.update(state="COMPLETED" if rc == 0 else "FAILED", end=time.time(), exit=rc); _save(job)
+
+
+def _ids(args):
+    for i, a in enumerate(args):
+        if a == "-j" and i + 1 < len(args):
+            return [int(x) for x in args[i + 1].split(",") if x]
+        if a.startswith("--jobs="):
+            return [int(x) for x in a[7:].split(",") if x]
+    return sorted(int(p.stem) for p in JOBS.glob("*.json")) if JOBS.exists() else []
+
+
+def _elapsed(job):
+    if not job["start"]:
+        return "00:00:00"
+    s = int((job["end"] or time.time()) - job["start"])
+    return f"{s // 3600:02d}:{s % 3600 // 60:02d}:{s % 60:02d}"
+
+
+def squeue(args):
+    for jid in _ids(args):
+        try:
+            j = _load(jid)
+        except FileNotFoundError:
+            continue
+        if j["state"] in ("PENDING", "RUNNING"):
+            print(f"{jid} {j['state']} {j['node'] if j['state'] == 'RUNNING' else '(None)'}")
+
+
+def sacct(args):
+    for jid in _ids(args):
+        try:
+            j = _load(jid)
+        except FileNotFoundError:
+            continue
+        print(f"{jid}|{j['state']}|{_elapsed(j)}")
+
+
+def scancel(args):
+    for a in args:
+        if not a.isdigit():
+            continue
+        j = _load(int(a))
+        if j["state"] in ("PENDING", "RUNNING"):
+            if j.get("pid"):
+                try:
+                    os.killpg(j["pid"], 15)
+                except ProcessLookupError:
+                    pass
+            j.update(state="CANCELLED", end=time.time()); _save(j)
+
+
+if __name__ == "__main__":
+    cmd, rest = sys.argv[1], sys.argv[2:]
+    {"sbatch": sbatch, "squeue": squeue, "sacct": sacct, "scancel": scancel,
+     "_run": lambda a: _run(int(a[0]))}[cmd](rest)

everyome/core/engine.py

@@ -0,0 +1,78 @@
+"""Base class for interactive module engines (generalised from SPLiTscape part 2).
+
+An engine runs inside a session on a compute node (or this computer for the
+demo). It works on one run folder, keeps its decisions in session/state.json,
+and records every decision in order so the run can be written up later.
+Long steps run in the background and report progress.
+
+Subclasses list the methods the GUI may call in API; nothing else is reachable.
+Methods listed in REPLAY change the analysis; each call is logged with its
+arguments, so the session can be replayed exactly to repeat the run.
+"""
+from __future__ import annotations
+
+import json
+import threading
+import time
+import traceback
+from pathlib import Path
+
+
+class BaseEngine:
+    API: tuple[str, ...] = ()
+    REPLAY: tuple[str, ...] = ()
+
+    def __init__(self, run_dir: str):
+        self.dir = Path(run_dir)
+        self.out = self.dir / "session"
+        self.out.mkdir(exist_ok=True)
+        (self.dir / "results").mkdir(exist_ok=True)
+        self.run = json.loads((self.dir / "run.json").read_text())
+        self.params = json.loads((self.dir / "params.json").read_text())
+        self.state_path = self.out / "state.json"
+        self.state = json.loads(self.state_path.read_text()) if self.state_path.exists() else {"step": "start", "history": []}
+        self.task = {"running": False, "step": "", "progress": 0, "error": None, "message": ""}
+        self.lock = threading.Lock()
+        self._state_lock = threading.RLock()   # Intent: the call log and background steps both write state.json
+
+    # ---------------------------------------------------------------- state
+    def _save_state(self):
+        with self._state_lock:
+            self.state_path.write_text(json.dumps(self.state, indent=1))
+
+    def _record(self, action: str, **details):
+        """Note a decision (in order) for the run record and the lab-book protocol."""
+        with self._state_lock:
+            self.state.setdefault("history", []).append({"at": time.strftime("%Y-%m-%d %H:%M:%S"), "action": action, **details})
+            self._save_state()
+
+    def _log_call(self, method: str, args: dict):
+        """Keep the exact call (method and arguments) for replaying the session later."""
+        with self._state_lock:
+            self.state.setdefault("calls", []).append({"at": time.strftime("%Y-%m-%d %H:%M:%S"), "method": method, "args": args})
+            self._save_state()
+
+    def status(self):
+        return {"step": self.state.get("step"), "task": self.task, "module": self.run["module"], "run": self.run["id"]}
+
+    # ---------------------------------------------------------------- background work
+    def _background(self, fn, *args):
+        if self.task["running"]:
+            raise RuntimeError("Another step is still running.")
+        self.task = {"running": True, "step": fn.__name__, "progress": 0, "error": None, "message": ""}
+
+        def wrap():
+            try:
+                with self.lock:
+                    fn(*args)
+            except Exception as e:                       # report, never crash the session
+                self.task["error"] = f"{e}"
+                traceback.print_exc()
+            finally:
+                self.task["running"] = False
+                self.task["progress"] = 100 if not self.task["error"] else self.task["progress"]
+        threading.Thread(target=wrap, daemon=True).start()
+        return self.task
+
+    def _prog(self, p, msg):
+        self.task.update(progress=p, message=msg)