msp-sc 0.2.0__py3-none-any.whl

msp/__init__.py

@@ -0,0 +1,35 @@
+"""msp (multi-sample-pipeline): integrate osp per-sample outputs (harmony)
+→ multi-resolution leiden + UMAP → cluster QC / DEG tables → self-contained
+HTML report, with two optional Claude-agent steps that run afterwards:
+
+    integrate  (msp.integrate)  propose-only: nothing deleted, nothing named
+    inspect    (msp.inspect)    per-cluster five-test QC verdicts, proposals only
+    annotate   (msp.annotate)   coarse/fine cell identity on msp_leiden_r2.0,
+                                explicit merges, REAL removal → annotated.h5ad
+
+Entry points:
+    from msp import run_multi_sample_pipeline, generate_report
+
+    run_multi_sample_pipeline(["A/clustered.h5ad", "B/clustered.h5ad"],
+                              batch_col="project", outdir="msp_out")
+    generate_report("msp_out")
+
+Command line:
+    python -m msp A/clustered.h5ad B/clustered.h5ad --batch-col project --outdir msp_out
+    python -m msp ... --inspect --annotate --model claude-sonnet-5   # full chain
+    python -m msp.inspect  msp_out          # QC inspection agent only
+    python -m msp.annotate msp_out          # annotation agent only (after inspect)
+    python -m msp.report   msp_out          # rebuild the report only
+
+msp.inspect / msp.annotate are intentionally not imported here — they depend
+on the optional claude-agent-sdk (`pip install "msp[agent]"`); use
+`from msp.inspect import inspect_clusters` / `from msp.annotate import
+annotate_clusters` when needed.
+"""
+
+from .integrate import integrate_adata, load_and_merge, run_multi_sample_pipeline
+from .plots import save_single_umap
+from .report import generate_report
+
+__all__ = ["integrate_adata", "load_and_merge", "run_multi_sample_pipeline", "generate_report",
+           "save_single_umap"]

msp/__main__.py

@@ -0,0 +1,122 @@
+"""python -m msp: integrate osp per-sample outputs (concat → harmony → leiden →
+UMAP → QC/DEG tables → HTML report), end to end.
+
+With --inspect, the per-cluster QC inspection agent (msp.inspect) runs
+afterwards; with --annotate, the cell-type annotation agent (msp.annotate)
+runs after that (both need the optional claude-agent-sdk). Each step is
+skipped when its contract file already exists, so re-running the same
+command resumes where it stopped; --force redoes everything.
+"""
+
+import argparse
+import os
+import sys
+
+from .integrate import integrate_adata, run_multi_sample_pipeline
+from .report import generate_report, write_report_context
+
+parser = argparse.ArgumentParser(prog="msp", description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+parser.add_argument("inputs", nargs="*", help="per-sample clustered.h5ad files (osp outputs)")
+parser.add_argument("--from-h5ad", default=None, metavar="H5AD",
+                    help="instead of per-sample inputs: one already-merged h5ad with layers['counts'] "
+                         "(e.g. a previous round's annotated_zmip.h5ad) — re-integrated from scratch via "
+                         "integrate_adata; prior obs columns ride along as annotation evidence")
+parser.add_argument("--batch-col", required=True, help="obs column naming the sample/batch")
+parser.add_argument("--outdir", required=True)
+parser.add_argument("--species", default=None, help="stored in uns['msp']; context for the agents")
+parser.add_argument("--resolutions", type=float, nargs="+", default=[0.3, 1.0, 2.0],
+                    help="leiden resolutions; 1.0 and 2.0 must be present for inspect/annotate")
+parser.add_argument("--n-top-genes", type=int, default=2000)
+parser.add_argument("--n-pcs", type=int, default=50)
+parser.add_argument("--n-neighbors", type=int, default=15)
+parser.add_argument("--harmony", action="append", default=[], metavar="KEY=VALUE",
+                    help="harmonypy.run_harmony override, repeatable: e.g. --harmony theta=1 "
+                         "--harmony lamb=-1 --harmony max_iter_harmony=20 --harmony sigma=0.2 "
+                         "(defaults: theta=2, lamb=1, sigma=0.1, nclust=min(N/30,100), "
+                         "max_iter_harmony=10, max_iter_kmeans=20)")
+parser.add_argument("--inspect", action="store_true",
+                    help="after integration, run the per-cluster QC inspection agent (msp.inspect)")
+parser.add_argument("--annotate", action="store_true",
+                    help="after inspection, run the cell-type annotation agent (msp.annotate); "
+                         "implies --inspect")
+parser.add_argument("--language", default="English", help='agent prose language (default "English")')
+parser.add_argument("--model", default=None, help='model for the agents, e.g. "claude-sonnet-5"')
+parser.add_argument("--effort", default=None, choices=["low", "medium", "high", "xhigh", "max"],
+                    help="reasoning effort for the agents (models that support it)")
+parser.add_argument("--max-turns", type=int, default=None,
+                    help="agent turn budget (defaults: inspect 100, annotate 200)")
+parser.add_argument("--report-context", default=None, metavar="TEXT",
+                    help='where this run sits, for report titles (e.g. "round 2 · fu2022-meniscus"); '
+                         "persisted in <outdir>/report_context.txt so later report refreshes keep it")
+parser.add_argument("--force", action="store_true", help="redo steps whose outputs already exist")
+args = parser.parse_args()
+
+if bool(args.inputs) == bool(args.from_h5ad):
+    sys.exit("give either per-sample inputs or --from-h5ad, not both / neither")
+if args.annotate:
+    args.inspect = True
+if args.inspect and not {1.0, 2.0} <= set(args.resolutions):
+    sys.exit("--inspect/--annotate need leiden resolutions 1.0 and 2.0 (see --resolutions)")
+
+out = args.outdir
+write_report_context(out, args.report_context)
+
+
+def _parse_kv(items):
+    """KEY=VALUE → {key: number|list|str}; comma-separated values become lists."""
+    def conv(v):
+        for cast in (int, float):
+            try:
+                return cast(v)
+            except ValueError:
+                pass
+        return v
+    out = {}
+    for it in items:
+        if "=" not in it:
+            sys.exit(f"--harmony expects KEY=VALUE, got {it!r}")
+        k, v = it.split("=", 1)
+        out[k.strip()] = [conv(x) for x in v.split(",")] if "," in v else conv(v)
+    return out
+
+
+harmony_kwargs = _parse_kv(args.harmony)
+
+
+def _done(*names):
+    return all(os.path.exists(os.path.join(out, n)) for n in names)
+
+
+if args.force or not _done("integrated.h5ad", "report.html"):
+    kw = dict(species=args.species, resolutions=tuple(args.resolutions), n_top_genes=args.n_top_genes,
+              n_pcs=args.n_pcs, n_neighbors=args.n_neighbors, harmony_kwargs=harmony_kwargs)
+    if args.from_h5ad:
+        import scanpy as sc
+
+        ad = sc.read_h5ad(args.from_h5ad)
+        _, summary = integrate_adata(ad, args.batch_col, out, inputs=[args.from_h5ad], **kw)
+    else:
+        _, summary = run_multi_sample_pipeline(args.inputs, batch_col=args.batch_col, outdir=out, **kw)
+    print(summary)
+    print(f"report: {generate_report(out)}")
+else:
+    print(f"[resume] integration already done in {out} (integrated.h5ad + report.html) — skipping")
+
+agent_kw = dict(species=args.species, language=args.language, model=args.model, effort=args.effort)
+
+if args.inspect:
+    if args.force or not _done("inspection_proposal.json"):
+        from .inspect import inspect_clusters
+
+        inspect_clusters(out, max_turns=args.max_turns or 100, **agent_kw)
+    else:
+        print(f"[resume] inspection_proposal.json exists in {out} — skipping inspect")
+
+if args.annotate:
+    if args.force or not _done("annotation_proposal.json", "annotated.h5ad"):
+        from .annotate import annotate_clusters
+
+        annotate_clusters(out, max_turns=args.max_turns or 200, **agent_kw)
+    else:
+        print(f"[resume] annotation_proposal.json + annotated.h5ad exist in {out} — skipping annotate")

msp/agent_util.py

@@ -0,0 +1,70 @@
+"""Shared agent-call helper for every claude-agent-sdk session in msp / zmip.
+
+The CLI itself retries transient API errors (429 / overloaded / 5xx) with
+backoff, but a subscription usage window that is used up ("Claude usage
+limit reached …") ends the session with an error result and the CLI does
+NOT wait for the reset. A self-driving loop must not stop for that: this
+wrapper recognises limit-type failures and re-runs the whole query after a
+wait (session state is in-memory on the host — submitted entries persist,
+the agent simply starts its investigation again), bounded by a total wait
+budget. Any other failure is raised immediately.
+
+    async for message in run_query(prompt, options, label="inspect"):
+        ...
+
+Env: AGENT_LIMIT_WAIT_MIN (minutes between retries, default 10),
+AGENT_LIMIT_WAIT_MAX_H (total hours to keep waiting, default 12).
+"""
+
+import asyncio
+import os
+import re
+import time
+
+LIMIT_PATTERN = re.compile(
+    r"usage limit|rate[ _-]?limit|limit will reset|resets at|too many requests|overloaded|"
+    r"quota|429|capacity|out of extra usage|spend limit",
+    re.IGNORECASE,
+)
+
+
+def is_limit_error(text) -> bool:
+    return bool(text) and bool(LIMIT_PATTERN.search(str(text)))
+
+
+class AgentLimitExhausted(RuntimeError):
+    pass
+
+
+async def run_query(prompt, options, label="agent"):
+    """Yield the SDK's messages exactly like query(); if the run ends in a
+    limit-type error, wait and start over (bounded)."""
+    from claude_agent_sdk import ResultMessage, query
+
+    wait_min = float(os.environ.get("AGENT_LIMIT_WAIT_MIN", "10"))
+    max_h = float(os.environ.get("AGENT_LIMIT_WAIT_MAX_H", "12"))
+    waited = 0.0
+    attempt = 0
+    while True:
+        attempt += 1
+        limit_hit = None
+        try:
+            async for message in query(prompt=prompt, options=options):
+                if isinstance(message, ResultMessage) and getattr(message, "is_error", False) \
+                        and is_limit_error(getattr(message, "result", "")):
+                    limit_hit = message.result
+                    continue  # swallow: the retry below replaces this result
+                yield message
+        except Exception as e:  # transport-level failure carrying a limit message
+            if not is_limit_error(str(e)):
+                raise
+            limit_hit = str(e)
+        if limit_hit is None:
+            return
+        if waited / 3600 >= max_h:
+            raise AgentLimitExhausted(f"[{label}] usage limit still in force after {waited / 3600:.1f} h: {limit_hit}")
+        print(f"== [{label}] usage/rate limit (attempt {attempt}): {str(limit_hit)[:160]!r} — "
+              f"waiting {wait_min:.0f} min, {max_h - waited / 3600:.1f} h of wait budget left", flush=True)
+        t0 = time.time()
+        await asyncio.sleep(wait_min * 60)
+        waited += time.time() - t0