omicstra 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
omicstra/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """omicstra - cross-modal biomedical embedding alignment + reasoning via MCP."""
2
+
3
+ __version__ = "0.0.1"
@@ -0,0 +1,5 @@
1
+ """cohort adapters - the one place that knows a file layout.
2
+
3
+ each adapter produces AnnData in the shape the EDA steps expect, so no step
4
+ ever learns a cohort's storage format.
5
+ """
@@ -0,0 +1,147 @@
1
+ """adapter: Wang et al. original-ST cohort -> AnnData.
2
+
3
+ an adapter's only job is to produce AnnData in the shape the EDA steps expect.
4
+ it is the one place that knows a cohort's file layout, so no step ever learns
5
+ a file format.
6
+
7
+ this cohort stores raw integer counts in `selection.RData` as a matrix `cnts`
8
+ (observations x genes, versioned Ensembl IDs) plus a `spots` data.frame carrying
9
+ pixel coordinates. the sibling `rawCountsMatrices/` directory holds normalised
10
+ and batch-corrected floats despite its name - the EDA count_statistics step
11
+ tests for this rather than trusting either name.
12
+
13
+ requires Rscript on PATH. pyreadr cannot read R matrix objects, and rpy2 needs a
14
+ matching R version, so a subprocess is the reliable path.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import io
19
+ import subprocess
20
+ from pathlib import Path
21
+
22
+ import numpy as np
23
+ import pandas as pd
24
+
25
+ _R_READ = 'load("{path}"); write.csv({obj}, stdout())'
26
+
27
+
28
+ def _read_rdata(path: Path, obj: str) -> pd.DataFrame:
29
+ r = subprocess.run(["Rscript", "-e", _R_READ.format(path=path, obj=obj)],
30
+ capture_output=True, text=True)
31
+ if r.returncode != 0:
32
+ raise RuntimeError(f"Rscript failed reading {obj} from {path}: {r.stderr[:300]}")
33
+ if not r.stdout.strip():
34
+ raise RuntimeError(f"Rscript returned nothing for {obj} in {path}")
35
+ return pd.read_csv(io.StringIO(r.stdout), index_col=0)
36
+
37
+
38
+ def list_samples(by_array: str | Path) -> list[tuple[str, str]]:
39
+ """(slide, position) for every sample carrying selection.RData."""
40
+ root = Path(by_array)
41
+ out = []
42
+ for slide in sorted(p for p in root.iterdir() if p.is_dir()):
43
+ for pos in sorted(p for p in slide.iterdir() if p.is_dir()):
44
+ if (pos / "selection.RData").exists():
45
+ out.append((slide.name, pos.name))
46
+ return out
47
+
48
+
49
+ _R_SPARSE = '''
50
+ suppressMessages(library(Matrix))
51
+ load("{src}")
52
+ m <- as(as(as(cnts, "dMatrix"), "generalMatrix"), "CsparseMatrix")
53
+ writeMM(m, "{d}/counts.mtx")
54
+ writeLines(rownames(cnts), "{d}/rows.txt")
55
+ writeLines(colnames(cnts), "{d}/cols.txt")
56
+ write.csv(spots, "{d}/spots.csv")
57
+ '''
58
+
59
+
60
+ def _read_sparse(path: Path):
61
+ """counts as Matrix Market + coords as a small CSV.
62
+
63
+ the matrix is ~16% dense, so writing triplets instead of a dense CSV grid is
64
+ about 6x faster end to end (12s -> 2s per sample). falls back to the CSV path
65
+ if R's Matrix package is unavailable.
66
+ """
67
+ import tempfile
68
+
69
+ import scipy.io as sio
70
+
71
+ with tempfile.TemporaryDirectory() as d:
72
+ r = subprocess.run(["Rscript", "-e", _R_SPARSE.format(src=path, d=d)],
73
+ capture_output=True, text=True)
74
+ if r.returncode != 0:
75
+ raise RuntimeError(f"sparse read failed for {path}: {r.stderr[:300]}")
76
+ X = sio.mmread(f"{d}/counts.mtx").tocsr()
77
+ rows = Path(f"{d}/rows.txt").read_text().rstrip("\n").split("\n")
78
+ cols = Path(f"{d}/cols.txt").read_text().rstrip("\n").split("\n")
79
+ coords = pd.read_csv(f"{d}/spots.csv", index_col=0)
80
+ return X, rows, cols, coords
81
+
82
+
83
+ def load_sample(by_array: str | Path, slide: str, position: str,
84
+ gene_map: dict[str, str] | None = None, sparse: bool = True):
85
+ """one sample -> AnnData with raw counts, spatial coords, and a sample id."""
86
+ import anndata as ad
87
+ from scipy.sparse import csr_matrix
88
+
89
+ p = Path(by_array) / slide / position / "selection.RData"
90
+
91
+ if sparse:
92
+ try:
93
+ X, obs_names, var_ids, coords = _read_sparse(p)
94
+ except Exception:
95
+ sparse = False
96
+ if not sparse:
97
+ counts = _read_rdata(p, "cnts")
98
+ coords = _read_rdata(p, "spots")
99
+ X = csr_matrix(counts.values.astype(np.float32))
100
+ obs_names, var_ids = [str(i) for i in counts.index], list(counts.columns)
101
+
102
+ # keep only observations that carry coordinates
103
+ keep = [i for i, o in enumerate(obs_names) if o in set(coords.index)]
104
+ X = X[keep]
105
+ obs_names = [obs_names[i] for i in keep]
106
+ coords = coords.loc[obs_names]
107
+
108
+ var_names = ([gene_map.get(str(c).split(".")[0], str(c)) for c in var_ids]
109
+ if gene_map else [str(c) for c in var_ids])
110
+
111
+ a = ad.AnnData(X.astype(np.float32))
112
+ a.var_names = var_names
113
+ a.var_names_make_unique()
114
+ a.var["ensembl_id"] = [str(c).split(".")[0] for c in var_ids]
115
+ a.obs_names = obs_names
116
+ a.obs["sample"] = f"{slide}_{position}"
117
+
118
+ xcol = "pixel_x" if "pixel_x" in coords.columns else coords.columns[0]
119
+ ycol = "pixel_y" if "pixel_y" in coords.columns else coords.columns[1]
120
+ a.obsm["spatial"] = coords[[xcol, ycol]].to_numpy(dtype=float)
121
+ return a
122
+
123
+
124
+ def load_gene_map(path: str | Path) -> dict[str, str]:
125
+ df = pd.read_csv(path, sep="\t")
126
+ return dict(zip(df.iloc[:, 0].astype(str), df.iloc[:, 1].astype(str)))
127
+
128
+
129
+ def load_cohort(by_array: str | Path, samples: list[tuple[str, str]] | None = None,
130
+ gene_map_path: str | Path | None = None, max_samples: int | None = None):
131
+ """concatenate samples into one AnnData on the intersection of their gene sets."""
132
+ import anndata as ad
133
+
134
+ gene_map = load_gene_map(gene_map_path) if gene_map_path else None
135
+ picks = samples if samples is not None else list_samples(by_array)
136
+ if max_samples:
137
+ picks = picks[:max_samples]
138
+
139
+ parts = []
140
+ for slide, pos in picks:
141
+ try:
142
+ parts.append(load_sample(by_array, slide, pos, gene_map))
143
+ except Exception as e: # a dead sample must not kill the cohort load
144
+ print(f" skipped {slide}/{pos}: {type(e).__name__}: {e}")
145
+ if not parts:
146
+ raise RuntimeError("no samples loaded")
147
+ return ad.concat(parts, join="inner", index_unique="-")
@@ -0,0 +1,6 @@
1
+ """langgraph nodes - orchestrator, he_agent, st_agent, alignment_agent, eval_agent.
2
+
3
+ phase 1 lands graph.py here: a StateGraph wiring orchestrator -> he ‖ st -> join
4
+ -> alignment -> eval, traced in langsmith. on the read path the agents resolve
5
+ cached refs (no compute); on the compute path they trigger pipeline stages.
6
+ """
@@ -0,0 +1,202 @@
1
+ """modality-specialised agents - one per representation.
2
+
3
+ each agent owns exactly one encoder and answers one question: what does MY
4
+ modality contribute to THIS biological question, on this cohort? it reports the
5
+ encoder's provenance, the modality-only baseline where the cohort declares one,
6
+ and the guard it is responsible for. it does not align, evaluate, or route -
7
+ those belong to the judge (CLAUDE.md agent constraints).
8
+
9
+ read path: every embedding is already computed and cached. an agent resolves
10
+ references and reads recorded evidence; none of them touch a GPU.
11
+
12
+ the emission is a DiagnosticRecord, so a modality report is the same scientific
13
+ unit as an EDA step: method, scope, observed, criterion, result, decision.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ from omicstra.contracts.project import ProjectConfig
18
+ from omicstra.records import DiagnosticRecord
19
+ from omicstra.routing import _blocks, load_routing_evidence
20
+
21
+
22
+ def _encoder(cfg: ProjectConfig, name: str) -> dict:
23
+ e = next((e for e in cfg.encoders if e.name == name), None)
24
+ return e.model_dump() if e else {}
25
+
26
+
27
+ def _modality_baseline(task: dict, bar_id: str) -> dict | None:
28
+ """the modality-alone value this cohort recorded for the task, if any.
29
+
30
+ a floor or reference named for a raw modality IS that modality's solo
31
+ performance - the agent reads its own baseline out of the shared evidence
32
+ rather than keeping a second copy that could drift.
33
+ """
34
+ for blk in _blocks(task):
35
+ for key in ("floor", "reference"):
36
+ bar = blk.get(key)
37
+ if bar and bar.get("id") == bar_id:
38
+ # spread first, then set - the evidence file may itself carry a
39
+ # `role` note, and it must not be shadowed by, or shadow, ours
40
+ return {**bar, "metric": blk.get("metric", ""), "declared_as": key}
41
+ return None
42
+
43
+
44
+ def _separable(a: dict, b: dict) -> bool | None:
45
+ """do two candidates' intervals separate them? None when either has no CI.
46
+
47
+ the same rule the router applies. an ablation that sits inside the leader's
48
+ interval has not been shown to matter, however far apart the point estimates
49
+ look - and on a task the router already called a tie, claiming otherwise
50
+ would contradict the route from inside its own evidence.
51
+ """
52
+ ca, cb = a.get("ci"), b.get("ci")
53
+ if not ca or not cb:
54
+ return None
55
+ return not (ca[0] <= cb[1] and cb[0] <= ca[1])
56
+
57
+
58
+ def _report(step_id: str, method: str, scope: str, observed: dict, criterion: str,
59
+ result: str, decision: str, caveats: list[str]) -> DiagnosticRecord:
60
+ return DiagnosticRecord(step_id=step_id, method=method, scope=scope,
61
+ observed=observed, criterion=criterion, result=result,
62
+ decision=decision, caveats=caveats, actor="deterministic")
63
+
64
+
65
+ def _n_test(ev: dict) -> str:
66
+ return str((ev.get("eval_scope") or {}).get("test_niches", "?"))
67
+
68
+
69
+ # --- H&E -------------------------------------------------------------------
70
+ def he_agent(task_id: str, project_id: str | None = None) -> DiagnosticRecord:
71
+ """morphology. owns Virchow2 and the raw-H&E baseline."""
72
+ cfg, ev = ProjectConfig.load(project_id), load_routing_evidence(project_id)
73
+ enc = _encoder(cfg, "virchow2")
74
+ task = (ev.get("tasks") or {}).get(task_id, {})
75
+ base = _modality_baseline(task, "raw_he")
76
+
77
+ obs = {"encoder": enc.get("name"), "dim": enc.get("dim"),
78
+ "stain_normalised": False,
79
+ "tissue_fit": "validated - MSK breast cases in training",
80
+ "modality_alone": base}
81
+ if base:
82
+ res = (f"raw H&E alone scores {base['value']} on this task, declared as the "
83
+ f"{base['declared_as']}; alignment is measured against that")
84
+ dec = ("morphology carries a standalone signal here - any aligned method "
85
+ "must be reported against it")
86
+ else:
87
+ res = "this cohort declares no H&E-only baseline for this task"
88
+ dec = "cannot say what morphology contributes alone - report the aligned result only"
89
+ return _report(
90
+ f"he_agent::{task_id}",
91
+ "Virchow2 niche embedding, frozen foundation model, mean-pooled over the niche's tiles",
92
+ f"{_n_test(ev)} held-out niches, z_he view", obs,
93
+ "a modality-only baseline is reported wherever the cohort declares one",
94
+ res, dec,
95
+ ["Virchow2 is stain-invariant by construction - no Reinhard/Macenko and no "
96
+ "z-score before the foundation model (program.md hard constraint 2)"])
97
+
98
+
99
+ # --- spatial transcriptomics ------------------------------------------------
100
+ def st_agent(task_id: str, project_id: str | None = None) -> DiagnosticRecord:
101
+ """molecular expression. owns Novae and the raw-ST baseline."""
102
+ cfg, ev = ProjectConfig.load(project_id), load_routing_evidence(project_id)
103
+ enc = _encoder(cfg, "novae")
104
+ task = (ev.get("tasks") or {}).get(task_id, {})
105
+ base = _modality_baseline(task, "raw_st")
106
+
107
+ obs = {"encoder": enc.get("name"), "dim": enc.get("dim"),
108
+ "normalisation": "z-scored per subarray AFTER niche aggregation",
109
+ "tissue_fit": "validated_with_constraints - trained on subcellular "
110
+ "image-based ST, applied to 100um spots",
111
+ "modality_alone": base}
112
+ if base:
113
+ res = (f"raw ST alone scores {base['value']} on this task, declared as the "
114
+ f"{base['declared_as']}")
115
+ dec = ("expression alone is the weaker modality here - the lift over it is "
116
+ "the alignment claim")
117
+ else:
118
+ res = "this cohort declares no ST-only baseline for this task"
119
+ dec = "cannot say what expression contributes alone on this task"
120
+ return _report(
121
+ f"st_agent::{task_id}",
122
+ "Novae GNN niche latent, frozen, mean-pooled over the niche then z-scored per subarray",
123
+ f"{_n_test(ev)} held-out niches, z_st view", obs,
124
+ "a modality-only baseline is reported wherever the cohort declares one",
125
+ res, dec,
126
+ ["mc_weights are supervision only and never enter the ST feature vector "
127
+ "(program.md hard constraint 3)",
128
+ "Novae zero-shot is undefined below its prototype count - subarrays under "
129
+ "512 spots are excluded"])
130
+
131
+
132
+ # --- pathway ----------------------------------------------------------------
133
+ def pathway_agent(task_id: str, project_id: str | None = None) -> DiagnosticRecord:
134
+ """pathway context. owns gpath2vec, its sha-lock, and the ablation.
135
+
136
+ the only agent whose encoder reads knowledge from outside the measurement:
137
+ Virchow2 and Novae encode the assay, gpath2vec additionally reads a curated
138
+ Reactome hierarchy. that is why it needs an ablation to be load-bearing.
139
+ """
140
+ cfg, ev = ProjectConfig.load(project_id), load_routing_evidence(project_id)
141
+ enc = _encoder(cfg, "gpath2vec")
142
+ task = (ev.get("tasks") or {}).get(task_id, {})
143
+
144
+ # an ablation is only interpretable against a MATCHED control - a run
145
+ # differing in the ablated component and nothing else. comparing it to the
146
+ # grid leader attributes to one change an effect that two changes produced.
147
+ spec = (ev.get("ablations") or {}).get("gpath2vec", {})
148
+ abl_id = spec.get("run", "R6_v3")
149
+ ctl_id = spec.get("matched_control")
150
+ abl = ctl = None
151
+ metric, sep = "", None
152
+ for blk in _blocks(task):
153
+ by_id = {c["id"]: c for c in blk.get("candidates", [])}
154
+ if abl_id in by_id and ctl_id in by_id:
155
+ abl, ctl = by_id[abl_id], by_id[ctl_id]
156
+ sep = _separable(abl, ctl)
157
+ metric = blk.get("metric", "")
158
+ break
159
+
160
+ obs = {"encoder": enc.get("name"), "dim": enc.get("dim"),
161
+ "sha256": cfg.gpath2vec_sha256,
162
+ "reads_external_knowledge": "Reactome hierarchy, TF-restricted Fisher enrichment",
163
+ "ablation_run": abl_id, "matched_control": ctl_id,
164
+ "differs_only_in": spec.get("differs_only_in", ""),
165
+ "ablation": abl, "control": ctl, "intervals_separate": sep}
166
+ if abl is None or ctl is None:
167
+ res = ("this cohort does not score the ablation and its matched control on "
168
+ "this task")
169
+ dec = "cannot say whether the pathway anchor is load-bearing here"
170
+ else:
171
+ d = ctl["value"] - abl["value"]
172
+ res = (f"matched ablation on {metric[:52]}: removing gpath2vec moves "
173
+ f"{ctl['value']} to {abl['value']}, a difference of {d:+g}")
174
+ if sep is False:
175
+ dec = ("NOT shown load-bearing here - the ablation's interval overlaps its "
176
+ "matched control's, so removing the pathway anchor is not measurably "
177
+ "worse on this task")
178
+ elif sep is None:
179
+ dec = (f"the matched control is {d:+g} above the ablation, but neither "
180
+ "carries an interval - report the gap, do not call it separation"
181
+ if d else
182
+ "the ablation matches its control - the anchor is not load-bearing here")
183
+ else:
184
+ dec = ("load-bearing for this task - the ablation's interval sits clear of "
185
+ "its matched control's")
186
+ return _report(
187
+ f"pathway_agent::{task_id}",
188
+ "gpath2vec niche embedding - metapath2vec over a niche-pathway graph built "
189
+ "from TF-restricted Fisher enrichment against Reactome",
190
+ f"{_n_test(ev)} held-out niches", obs,
191
+ "the pathway anchor must be shown load-bearing by ablation, not assumed",
192
+ res, dec,
193
+ ["sha-locked to the v3 build - a silent swap invalidates every H3 result",
194
+ "TF restriction under-represents non-TF gene families (collagen, "
195
+ "immunoglobulin); this is why TLS signal degrades under late fusion",
196
+ "z_st and z_mean are circular when gpath2vec is on the ST input - z_he is "
197
+ "the only clean view for H3",
198
+ (spec.get("untested_run") or "")])
199
+
200
+
201
+ MODALITY_AGENTS = {"he_agent": he_agent, "st_agent": st_agent,
202
+ "pathway_agent": pathway_agent}
omicstra/artifacts.py ADDED
@@ -0,0 +1,30 @@
1
+ """artifact store - owns the runs/{project}/ schema. one place to change for v4.
2
+
3
+ the read path resolves cached refs here (a -resume-style cache hit); the compute
4
+ path writes here. keeping this the single schema-owner means a new run layout is
5
+ a one-file change, not a sweep.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import json
10
+
11
+ from omicstra.contracts.project import ProjectConfig
12
+
13
+
14
+ class ArtifactStore:
15
+ def __init__(self, cfg: ProjectConfig):
16
+ self.cfg = cfg
17
+ # cohort paths resolve against the PROJECT root, never the package
18
+ self.runs_dir = cfg.runs_path
19
+
20
+ def hypothesis_summary(self, hypothesis: str) -> dict:
21
+ # runs/{project}_v3/eval/{H1|H2|H3}/summary.json
22
+ p = self.runs_dir / "eval" / hypothesis / "summary.json"
23
+ return json.loads(p.read_text())
24
+
25
+ def run_config(self, run_id: str) -> dict:
26
+ return json.loads((self.runs_dir / run_id / "run_config.json").read_text())
27
+
28
+ def verify_gpath2vec_sha(self, manifest: dict) -> bool:
29
+ got = manifest.get("sources", {}).get("gpath2vec_sha256")
30
+ return got == self.cfg.gpath2vec_sha256