ecarsi 0.2.8__py3-none-any.whl

ecarsi/__init__.py

@@ -0,0 +1,72 @@
+"""ecarsi — pluggable-harness tooling for the eca-rsi curation loop.
+
+HARNESS env var selects the agent execution backend for every call in this
+package (see ecarsi.harness): 'openai' (default — OpenAI Agents SDK driving
+Doubao through Ark), 'deepseek' (DeepSeek Harness / dsh driving Doubao), or
+'claude' (claude_agent_sdk, spends Claude Code quota)."""
+
+from __future__ import annotations
+
+import os
+
+
+def model() -> str:
+    """Model for every agent call in this package: MODEL env, else the
+    HARNESS-appropriate default — a bare model name is never portable
+    across backends. Same rule as osp/msp/zmip's harness.default_model()."""
+    from .harness import default_model
+
+    return default_model()
+
+
+def agent_config() -> dict[str, str]:
+    """The two independent choices that must stay fixed within a run."""
+    from .harness import backend_name
+
+    return {"harness": backend_name(), "model": model()}
+
+
+def effective_or_requested(fn) -> dict[str, str]:
+    """{"harness", "model"} for a manifest field recorded *after* an agent
+    call: the AgentConfig that actually produced the result if `fn` stashed
+    one (the `.last_effective_config` convention, same idea as `.last_cost` --
+    set by the caller of run_agent(), read here), else the plain env
+    snapshot from agent_config(). With a fallback pool (AGENT_MODEL_POOL) in
+    play, these can differ; without one they are always the same value.
+
+    Only usable where the manifest field is a *record* of what happened, not
+    an input to a pre-call resume/identity decision -- persample's own
+    config is built and hashed before its agent call can run at all, so it
+    correctly keeps using agent_config() unconditionally.
+    """
+    cfg = getattr(fn, "last_effective_config", None)
+    return cfg.as_manifest() if cfg is not None else agent_config()
+
+
+def check_agent_config(recorded: dict, where: str) -> str | None:
+    """Compare a resumed stage's recorded {harness, model} against the
+    current selection; never blocks the resume (2026-09-10: switching
+    backend mid-run is a legitimate operator move -- round 3 shows the
+    current model's QC judgement is too lax, so the remaining rounds
+    continue with a stronger one -- dropped the --allow-agent-change guard
+    that used to require asking permission for it every time).
+
+    Returns the human-readable mismatch message (also printed) so a caller
+    that has a durable per-unit log can additionally record it there; None
+    when the config matches or predates harness/model recording (older
+    manifests remain resumable with a warning because their original choice
+    cannot be proved either way).
+    """
+    if "harness" not in recorded or "model" not in recorded:
+        print(f"[agent] {where} predates harness/model recording — resume cannot verify the old choice")
+        return None
+    want = agent_config()
+    got = {"harness": str(recorded["harness"]), "model": str(recorded["model"])}
+    if got == want:
+        return None
+    message = (
+        f"{where} used harness={got['harness']} model={got['model']}; current selection is "
+        f"harness={want['harness']} model={want['model']}"
+    )
+    print(f"[agent] {message} — continuing (mixed-model run)")
+    return message

ecarsi/__main__.py

@@ -0,0 +1,154 @@
+"""eca-rsi — the one entry point of the main line.
+
+    eca-rsi [--harness BACKEND] [--model MODEL] run <eca-pp-dir> <root> [--rounds N] [--cap 10] [--mirror DIR] [--serve [PORT]]
+    eca-rsi organize  <eca-pp-dir> <root> [--mirror DIR]
+    eca-rsi persample <unit> [...]           eca-rsi loop   <unit> [...]      (both also take --mirror DIR)
+    eca-rsi crosssample <unit> [round_dir]   eca-rsi zoomin <unit> [round_dir]
+    eca-rsi ledger    <unit> [round dirs]    eca-rsi index  <root|unit>
+    eca-rsi prune     <root|unit> [--dry-run]  (runs by itself after every release unless --no-prune)
+    eca-rsi serve     [dir...] [--registry F] [--port] [--ngrok --domain D] [--auth U:P]
+    eca-rsi serve     scan-add|remove|list|dump|reload ...   (edit the registry file)
+    eca-rsi umapdata  <h5ad> <out.json>
+
+`run` chains everything for one dataset: organize the eca-pp products into
+<root>, then for every analysis unit persample (osp) → loop (msp + zmip
+rounds until the cell count converges) → release; with --serve it finally
+adds <root> to the serve registry and serves it (and everything else in the
+registry) in the foreground at http://127.0.0.1:PORT/<root-name>/ (add
+--ngrok to publish; Ctrl-C to stop). --mirror DIR keeps a copy of <root>
+on long-term storage: light files (pages, logs, manifests, reports, figures)
+after every step, everything at release (ecarsi.mirror); DIR is remembered
+in <root>/mirror.json, so resumes and single steps keep mirroring.
+Every step resumes, so re-running the same command after an interruption
+continues where it stopped. `python -m ecarsi ...` is the same thing.
+The global --harness and --model options may appear before or after the
+subcommand; explicit CLI values override HARNESS / MODEL environment values.
+A resume with a different harness/model than an earlier stage used is
+allowed (switching mid-run to a stronger model is a legitimate operator
+move) and reported in progress.log / needs_review, never blocked.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from . import layout as L
+
+STEPS = ("organize", "persample", "crosssample", "zoomin", "loop", "ledger", "index", "serve", "umapdata", "prune")
+
+
+def _module(name: str):
+    import importlib
+
+    return importlib.import_module(f"ecarsi.{name}")
+
+
+def run(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(prog="eca-rsi run", description="organize → persample → loop for every unit (→ serve)")
+    ap.add_argument("input", help="eca-pp output directory (standardize/standardized.h5ad + result.json per sample)")
+    ap.add_argument("root", help="run root; everything lands under <root>/units/<unit>/ (ecarsi.layout)")
+    ap.add_argument("--stop-after", choices=["organize", "persample"], help="validate the front pipeline without entering the loop")
+    ap.add_argument("--plan-json", help="explicit organize plan")
+    ap.add_argument("--rounds", type=int, default=None, help="fixed number of loop rounds (default: converge on cell count)")
+    ap.add_argument("--cap", type=int, default=None, help="loop safety cap (default 10)")
+    ap.add_argument("--force-reopen", action="store_true", help="continue past an existing release")
+    ap.add_argument("--no-prune", action="store_true", help="keep intermediate round h5ads after release (default: prune them)")
+    ap.add_argument("--mirror", metavar="DIR", help="keep a copy of <root> here: light files after every step, all of it at release")
+    ap.add_argument("--serve", nargs="?", const=8899, type=int, default=None, metavar="PORT",
+                    help="after the run, add <root> to the serve registry and serve (foreground) on this port (default 8899)")
+    ap.add_argument("--ngrok", action="store_true", help="with --serve: also open an ngrok tunnel")
+    ap.add_argument("--domain", default=None, help="with --serve: reserved ngrok domain")
+    ap.add_argument("--auth", default=None, help="with --serve: web-level password USER:PASS")
+    a = ap.parse_args(argv)
+    root = Path(a.root).resolve()
+    if a.mirror:
+        _module("mirror").configure(root, a.mirror)  # remembered in <root>/mirror.json; every step reads it there
+
+    organize_args = [a.input, str(root)]
+    if a.plan_json:
+        organize_args += ["--plan-json", a.plan_json]
+    rc = _module("organize").main(organize_args)
+    if rc or a.stop_after == "organize":
+        return rc
+    units = L.units(root)
+    if not units:
+        print(f"[eca-rsi] no analysis units under {L.units_root(root)}")
+        return 3
+    print(f"[eca-rsi] {len(units)} unit(s): " + ", ".join(u.name for u in units))
+    loop_args = []
+    if a.rounds is not None:
+        loop_args += ["--rounds", str(a.rounds)]
+    if a.cap is not None:
+        loop_args += ["--cap", str(a.cap)]
+    if a.force_reopen:
+        loop_args.append("--force-reopen")
+    if a.no_prune:
+        loop_args.append("--no-prune")
+    failed = []
+    for u in units:
+        print(f"\n[eca-rsi] ===== unit {u.name}: persample =====", flush=True)
+        rc = _module("persample").main([str(u)])
+        if rc:
+            failed.append((u.name, "persample", rc))
+            continue
+        if a.stop_after == "persample":
+            continue
+        print(f"\n[eca-rsi] ===== unit {u.name}: loop =====", flush=True)
+        rc = _module("loop").main([str(u), *loop_args])
+        if rc:
+            failed.append((u.name, "loop", rc))
+    _module("index").write_all(root)
+    for name, step, rc in failed:
+        print(f"[eca-rsi] FAILED {name} at {step} (rc={rc}) — re-run the same command to resume")
+    if a.serve is not None:
+        # record this root in the registry file (so any later `eca-rsi serve`
+        # shows it too), then serve everything in the registry in the
+        # foreground until Ctrl-C — the server itself keeps no state
+        serve = _module("serve")
+        try:
+            serve.Registry(serve.default_registry()).bind(root.name, root)
+        except ValueError as e:
+            print(f"[eca-rsi] not added to the registry ({e}); serving it for this process only")
+        serve_args = [str(root), "--port", str(a.serve)]
+        if a.ngrok:
+            serve_args.append("--ngrok")
+        if a.domain:
+            serve_args += ["--domain", a.domain]
+        if a.auth:
+            serve_args += ["--auth", a.auth]
+        print(f"[eca-rsi] serving {root.name} at http://127.0.0.1:{a.serve}/{root.name}/", flush=True)
+        rc = serve.main(serve_args)
+        return rc or (1 if failed else 0)
+    print(f"\n[eca-rsi] done — landing page: {root / L.INDEX}  (eca-rsi serve scan-add {root}; eca-rsi serve)")
+    return 1 if failed else 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    from harness_bridge import configure_logging
+    configure_logging("ecarsi", stream=sys.stderr)
+    argv = sys.argv[1:] if argv is None else argv
+    runtime = argparse.ArgumentParser(add_help=False)
+    runtime.add_argument("--harness", choices=["deepseek", "openai", "claude"])
+    runtime.add_argument("--model")
+    selected, argv = runtime.parse_known_args(argv)
+    if selected.harness:
+        os.environ["HARNESS"] = selected.harness
+    if selected.model:
+        os.environ["MODEL"] = selected.model
+    if not argv or argv[0] in ("-h", "--help"):
+        print(__doc__)
+        return 0 if argv else 2
+    cmd, rest = argv[0], argv[1:]
+    if cmd == "run":
+        return run(rest)
+    if cmd in STEPS:
+        return _module(cmd).main(rest)
+    print(f"unknown command {cmd!r}; expected run or one of {', '.join(STEPS)}")
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ecarsi/agent_retry.py

@@ -0,0 +1,41 @@
+"""Retry wrapper for the cheap, side-effect-free agent-only calls (plan,
+sample-column identification): under concurrent Slurm job start, many
+`claude` CLI subprocesses initializing at once can blow the SDK's control
+handshake ("Control request timeout: initialize") or die with a transient
+connection error. These calls do nothing but read + return structured
+output, so a bare retry is safe — no partial state to clean up.
+
+Heavier steps (persample driving, crosssample, zoomin) are not wrapped here:
+they write real files and are already resumable at the eca-rsi step level
+(the sbatch wrapper retries `eca-rsi run` itself, which reuses that resume).
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Callable, Coroutine, TypeVar
+
+T = TypeVar("T")
+
+MAX_ATTEMPTS = 6
+BACKOFF_SECONDS = 20  # linear: 20s, 40s, 60s, 80s, 100s
+
+
+def run_with_retry(coro_fn: Callable[[], Coroutine[object, object, T]], label: str) -> T:
+    """asyncio.run(coro_fn()) with retries on transient SDK/agent failures."""
+    import asyncio
+
+    last_exc: Exception | None = None
+    for attempt in range(1, MAX_ATTEMPTS + 1):
+        try:
+            return asyncio.run(coro_fn())
+        except Exception as e:  # noqa: BLE001 - deliberately broad, see module docstring
+            last_exc = e
+            if attempt == MAX_ATTEMPTS:
+                break
+            wait = BACKOFF_SECONDS * attempt
+            print(f"[retry] {label} attempt {attempt}/{MAX_ATTEMPTS} failed "
+                  f"({type(e).__name__}: {e}); retrying in {wait}s")
+            time.sleep(wait)
+    assert last_exc is not None
+    raise last_exc

ecarsi/cost.py

@@ -0,0 +1,184 @@
+"""ecarsi.cost — agent spend and token usage, recorded per step in progress.log
+and summed at release.
+
+The harness prints one of two per-run lines to stdout (both in this process
+and inside the osp / msp / zmip kernels, which are subprocesses):
+`== [label] agent cost: $X` (claude, real dollars) or
+`== [label] ... N input / M output tokens ...` (openai/Doubao, no dollar figure
+but real token counts). Nothing persisted that; it scrolled by in the Slurm
+log. Now:
+
+- kernel subprocesses are run through `run_streamed()`, which echoes their
+  output unchanged and, on every cost/token line, appends a `cost ...` event
+  to the unit's progress.log;
+- ecarsi's own agent calls call `record()` with the harness result's
+  cost_usd/tokens_in/tokens_out;
+- `summarize()` reads those events back (progress.log is the audit trail
+  anyway) and release/summary.md gets an "Agent cost" section from it.
+
+A backend that reports neither (deepseek) simply produces no events — the
+section then says so instead of pretending zero.
+
+Separately, the bridge (agent-harness-bridge >= 0.2.8) prints one more line
+for every backend regardless of whether it reports cost: `== [label]
+resolved backend: harness=H model=M`. `run_streamed()` / persample's `_pump`
+also catch that line and record it (`backend_events` / `round_backends`) --
+this is how ecarsi finds out which model actually answered inside a kernel
+subprocess (osp per-sample worker, msp/zmip round subprocess) when a
+fallback pool is in play (eca-rsi#6); release/summary.md's "Backends per
+round" section (eca-rsi#5) is built from it.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from collections import OrderedDict
+from pathlib import Path
+
+from . import layout as L
+
+COST_RE = re.compile(r"(?:\[(?P<label>[^\]]*)\]\s*)?(?P<pre>[\w -]*?)\s*agent cost: \$(?P<usd>[0-9]+(?:\.[0-9]+)?)")
+TOKEN_RE = re.compile(r"\[(?P<label>[^\]]*)\].*?(?P<tin>\d+) input / (?P<tout>\d+) output tokens")
+EVENT_RE = re.compile(
+    r"^cost step=(?P<step>\S+)(?: usd=(?P<usd>[0-9.]+))?"
+    r"(?: tokens_in=(?P<tin>\d+))?(?: tokens_out=(?P<tout>\d+))?(?: label=(?P<label>.*))?$"
+)
+BACKEND_RE = re.compile(r"\[(?P<label>[^\]]*)\] resolved backend: harness=(?P<harness>\S+) model=(?P<model>\S+)")
+BACKEND_EVENT_RE = re.compile(
+    r"^agent step=(?P<step>\S+) harness=(?P<harness>\S+) model=(?P<model>\S+)(?: label=(?P<label>.*))?$"
+)
+ROUND_RE = re.compile(r"^round(\d+)")
+
+
+def record(unit: Path, step: str, usd: float | None, label: str = "",
+           tokens_in: int | None = None, tokens_out: int | None = None) -> None:
+    """One agent run's spend/usage -> progress.log (no-op when the backend gave neither)."""
+    if usd is None and tokens_in is None and tokens_out is None:
+        return
+    parts = [f"step={step}"]
+    if usd is not None:
+        parts.append(f"usd={usd:.4f}")
+    if tokens_in is not None:
+        parts.append(f"tokens_in={tokens_in}")
+    if tokens_out is not None:
+        parts.append(f"tokens_out={tokens_out}")
+    if label:
+        parts.append(f"label={label}")
+    L.log_event(unit, "cost " + " ".join(parts), echo=False)
+
+
+def record_backend(unit: Path, step: str, harness: str, model: str, label: str = "") -> None:
+    """One agent run's {harness, model} -> progress.log, independent of the
+    cost/token event above (a single call prints both a cost-or-token line
+    and this line; recording them separately avoids double-counting `n` in
+    summarize())."""
+    line = f"agent step={step} harness={harness} model={model}" + (f" label={label}" if label else "")
+    L.log_event(unit, line, echo=False)
+
+
+def _scan_line(unit: Path, step: str, line: str) -> None:
+    """Check one line of kernel subprocess stdout against every pattern this
+    module knows how to record; shared by run_streamed() and persample's
+    per-sample pump so both capture cost/token/backend lines the same way."""
+    m = COST_RE.search(line)
+    if m:
+        label = (m.group("label") or m.group("pre") or "").strip()
+        record(unit, step, float(m.group("usd")), label)
+    m = TOKEN_RE.search(line)
+    if m:
+        record(unit, step, None, m.group("label").strip(), int(m.group("tin")), int(m.group("tout")))
+    m = BACKEND_RE.search(line)
+    if m:
+        record_backend(unit, step, m.group("harness"), m.group("model"), m.group("label").strip())
+
+
+def run_streamed(cmd: str, unit: Path, step: str) -> int:
+    """subprocess.run(cmd, shell=True) with the output passed through line by
+    line and every harness cost/token/backend line also recorded against `step`."""
+    proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)
+    assert proc.stdout is not None
+    for line in proc.stdout:
+        print(line, end="", flush=True)
+        _scan_line(unit, step, line)
+    return proc.wait()
+
+
+def events(unit: Path) -> list[dict]:
+    out = []
+    for ts, ev in L.read_log(unit):
+        m = EVENT_RE.match(ev)
+        if m:
+            out.append({
+                "time": ts, "step": m.group("step"),
+                "usd": float(m.group("usd")) if m.group("usd") else None,
+                "tokens_in": int(m.group("tin")) if m.group("tin") else None,
+                "tokens_out": int(m.group("tout")) if m.group("tout") else None,
+                "label": m.group("label") or "",
+            })
+    return out
+
+
+def summarize(unit: Path) -> dict:
+    """{"total": float, "tokens_in": int, "tokens_out": int, "n": int,
+    "by_step": {step: {"usd", "tokens_in", "tokens_out", "n"}}} in first-seen step order."""
+    by: "OrderedDict[str, dict]" = OrderedDict()
+    total, tin_total, tout_total, n = 0.0, 0, 0, 0
+    for e in events(unit):
+        d = by.setdefault(e["step"], {"usd": 0.0, "tokens_in": 0, "tokens_out": 0, "n": 0})
+        d["usd"] += e["usd"] or 0.0
+        d["tokens_in"] += e["tokens_in"] or 0
+        d["tokens_out"] += e["tokens_out"] or 0
+        d["n"] += 1
+        total += e["usd"] or 0.0
+        tin_total += e["tokens_in"] or 0
+        tout_total += e["tokens_out"] or 0
+        n += 1
+    return {
+        "total": round(total, 2), "tokens_in": tin_total, "tokens_out": tout_total, "n": n,
+        "by_step": {k: {"usd": round(v["usd"], 2), "tokens_in": v["tokens_in"], "tokens_out": v["tokens_out"], "n": v["n"]}
+                    for k, v in by.items()},
+    }
+
+
+def backend_events(unit: Path) -> list[dict]:
+    out = []
+    for ts, ev in L.read_log(unit):
+        m = BACKEND_EVENT_RE.match(ev)
+        if m:
+            out.append({"time": ts, "step": m.group("step"), "harness": m.group("harness"),
+                        "model": m.group("model"), "label": m.group("label") or ""})
+    return out
+
+
+def round_backends(unit: Path) -> "OrderedDict[str, list[str]]":
+    """{round label ("round03" or "front" for pre-round steps): sorted
+    ["harness:model", ...] seen there}, in first-seen order. Distinct
+    configs within one round are normal (osp/msp/zmip can each resolve a
+    fallback pool differently) -- eca-rsi#5 wants this listed as fact, not
+    flagged as a mismatch the way check_agent_config's resume check is."""
+    by: "OrderedDict[str, set]" = OrderedDict()
+    for e in backend_events(unit):
+        m = ROUND_RE.match(e["step"])
+        key = m.group(0) if m else "front"
+        by.setdefault(key, set()).add(f"{e['harness']}:{e['model']}")
+    return OrderedDict((k, sorted(v)) for k, v in by.items())
+
+
+def backend_summary_md(unit: Path) -> list[str]:
+    by_round = round_backends(unit)
+    if not by_round:
+        return ["## Backends per round", "", "no backend reported (predates backend logging, or no fallback pool was in play)"]
+    rows = [f"| {k} | {', '.join(v)} |" for k, v in by_round.items()]
+    return ["## Backends per round", "", "| round | backend(s) used |", "|---|---|", *rows]
+
+
+def summary_md(unit: Path) -> list[str]:
+    s = summarize(unit)
+    if not s["n"]:
+        return ["## Agent cost", "", "no cost/token usage reported (backend does not report it, or the run predates cost logging)"]
+    rows = [f"| {step} | {v['n']} | ${v['usd']:.2f} | {v['tokens_in']} | {v['tokens_out']} |" for step, v in s["by_step"].items()]
+    return ["## Agent cost", "",
+            f"Total: **${s['total']:.2f}**, {s['tokens_in']} input / {s['tokens_out']} output tokens over "
+            f"{s['n']} agent run(s) — from `cost` events in progress.log", "",
+            "| step | agent runs | USD | tokens in | tokens out |", "|---|---|---|---|---|", *rows]