pen-stack 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. pen_stack/__init__.py +2 -0
  2. pen_stack/_resources.py +34 -0
  3. pen_stack/active/__init__.py +20 -0
  4. pen_stack/active/acquire.py +165 -0
  5. pen_stack/active/brains.py +74 -0
  6. pen_stack/active/campaign.py +109 -0
  7. pen_stack/active/design.py +66 -0
  8. pen_stack/active/validate.py +104 -0
  9. pen_stack/adapt/__init__.py +14 -0
  10. pen_stack/adapt/finetune.py +33 -0
  11. pen_stack/adapt/ingest.py +86 -0
  12. pen_stack/adapt/pipeline.py +101 -0
  13. pen_stack/adapt/recalibrate.py +58 -0
  14. pen_stack/adapt/report.py +130 -0
  15. pen_stack/agent/__init__.py +1 -0
  16. pen_stack/agent/cite.py +175 -0
  17. pen_stack/agent/co_scientist.py +262 -0
  18. pen_stack/agent/epistemic.py +102 -0
  19. pen_stack/agent/guardrails.py +67 -0
  20. pen_stack/agent/mcp_server.py +215 -0
  21. pen_stack/agent/orchestrator.py +112 -0
  22. pen_stack/agent/orchestrator_live.py +56 -0
  23. pen_stack/agent/pen_agent.py +242 -0
  24. pen_stack/agent/scope.py +60 -0
  25. pen_stack/agent/tools.py +130 -0
  26. pen_stack/api/__init__.py +12 -0
  27. pen_stack/api/manifest.py +160 -0
  28. pen_stack/atlas/__init__.py +1 -0
  29. pen_stack/atlas/atlas.parquet +0 -0
  30. pen_stack/atlas/build_wtkb.py +80 -0
  31. pen_stack/atlas/crosslink.py +179 -0
  32. pen_stack/atlas/expand.py +190 -0
  33. pen_stack/atlas/guide_design.py +178 -0
  34. pen_stack/atlas/schema.py +59 -0
  35. pen_stack/atlas/scorecard.py +134 -0
  36. pen_stack/atlas/scorecard_v3.parquet +0 -0
  37. pen_stack/atlas/universe.py +75 -0
  38. pen_stack/atlas/universe_v3.parquet +0 -0
  39. pen_stack/atlas/variant_propose.py +155 -0
  40. pen_stack/atlas/writer_efficiency.py +184 -0
  41. pen_stack/atlas/writer_predict.py +229 -0
  42. pen_stack/atlas/writer_recommend.py +170 -0
  43. pen_stack/atlas/writer_verify.py +167 -0
  44. pen_stack/atlas/wtkb.parquet +0 -0
  45. pen_stack/bridge/__init__.py +1 -0
  46. pen_stack/bridge/activity.py +52 -0
  47. pen_stack/bridge/cli.py +65 -0
  48. pen_stack/bridge/fold_qc.py +53 -0
  49. pen_stack/bridge/guide_qc.py +87 -0
  50. pen_stack/bridge/ingest.py +139 -0
  51. pen_stack/bridge/offtarget.py +191 -0
  52. pen_stack/bridge/offtarget_energetics.py +105 -0
  53. pen_stack/bridge/ortholog_screen.py +73 -0
  54. pen_stack/bridge/pipeline.py +83 -0
  55. pen_stack/build/__init__.py +16 -0
  56. pen_stack/build/cloudlab.py +74 -0
  57. pen_stack/build/ingest.py +47 -0
  58. pen_stack/build/protocol.py +82 -0
  59. pen_stack/build/simlab.py +30 -0
  60. pen_stack/cli.py +126 -0
  61. pen_stack/data/__init__.py +1 -0
  62. pen_stack/data/encode.py +84 -0
  63. pen_stack/data/genome.py +71 -0
  64. pen_stack/data/ingest_chromatin.py +119 -0
  65. pen_stack/data/ingest_integration.py +112 -0
  66. pen_stack/data/ingest_safety_annot.py +201 -0
  67. pen_stack/data/ingest_trip.py +76 -0
  68. pen_stack/design/__init__.py +14 -0
  69. pen_stack/design/capsid_generate.py +62 -0
  70. pen_stack/design/generate.py +70 -0
  71. pen_stack/design/pareto.py +70 -0
  72. pen_stack/design/space.py +137 -0
  73. pen_stack/design/writer_variants.py +121 -0
  74. pen_stack/env/__init__.py +1 -0
  75. pen_stack/env/genome_writing_env.py +248 -0
  76. pen_stack/env/policies.py +94 -0
  77. pen_stack/graph/__init__.py +21 -0
  78. pen_stack/graph/build.py +133 -0
  79. pen_stack/graph/cell_types.py +58 -0
  80. pen_stack/graph/ingest.py +132 -0
  81. pen_stack/graph/query.py +148 -0
  82. pen_stack/graph/schema.py +100 -0
  83. pen_stack/loop/__init__.py +15 -0
  84. pen_stack/loop/continual.py +61 -0
  85. pen_stack/loop/cycle.py +84 -0
  86. pen_stack/loop/drift.py +41 -0
  87. pen_stack/mech/__init__.py +1 -0
  88. pen_stack/mech/classify_atlas.py +71 -0
  89. pen_stack/mech/pfam_whitelist.yaml +247 -0
  90. pen_stack/mech/whitelist.py +66 -0
  91. pen_stack/monitor/__init__.py +1 -0
  92. pen_stack/monitor/europepmc.py +32 -0
  93. pen_stack/monitor/run.py +57 -0
  94. pen_stack/monitor/triage.py +63 -0
  95. pen_stack/oracles/__init__.py +65 -0
  96. pen_stack/oracles/affinity.py +116 -0
  97. pen_stack/oracles/cache.py +53 -0
  98. pen_stack/oracles/energetics.py +33 -0
  99. pen_stack/oracles/genome.py +167 -0
  100. pen_stack/oracles/protein_design.py +136 -0
  101. pen_stack/oracles/reliability.py +64 -0
  102. pen_stack/oracles/rna.py +28 -0
  103. pen_stack/oracles/schema.py +77 -0
  104. pen_stack/oracles/status.py +123 -0
  105. pen_stack/oracles/structure.py +42 -0
  106. pen_stack/oracles/structure_run.py +76 -0
  107. pen_stack/oracles/vcell.py +74 -0
  108. pen_stack/planner/__init__.py +1 -0
  109. pen_stack/planner/ada_risk.py +64 -0
  110. pen_stack/planner/antipeg_oracle.py +75 -0
  111. pen_stack/planner/capsid_epitope_oracle.py +135 -0
  112. pen_stack/planner/cargo.py +56 -0
  113. pen_stack/planner/cargo_polish.py +146 -0
  114. pen_stack/planner/chromosome.py +106 -0
  115. pen_stack/planner/delivery.py +55 -0
  116. pen_stack/planner/delivery_constraints.py +110 -0
  117. pen_stack/planner/delivery_immune.py +61 -0
  118. pen_stack/planner/delivery_immunology.py +222 -0
  119. pen_stack/planner/delivery_predict.py +196 -0
  120. pen_stack/planner/delivery_vehicles.py +37 -0
  121. pen_stack/planner/genotoxicity_oracle.py +112 -0
  122. pen_stack/planner/immune_mhc2.py +154 -0
  123. pen_stack/planner/immune_profile.py +292 -0
  124. pen_stack/planner/innate_sensing.py +135 -0
  125. pen_stack/planner/multiplex.py +110 -0
  126. pen_stack/planner/optimize.py +278 -0
  127. pen_stack/planner/pipeline.py +87 -0
  128. pen_stack/planner/report.py +26 -0
  129. pen_stack/planner/router.py +57 -0
  130. pen_stack/planner/seroprevalence_oracle.py +92 -0
  131. pen_stack/planner/target_site.py +118 -0
  132. pen_stack/rag/__init__.py +1 -0
  133. pen_stack/rag/corpus.py +133 -0
  134. pen_stack/rag/embed.py +98 -0
  135. pen_stack/rag/ground.py +131 -0
  136. pen_stack/rag/index.py +53 -0
  137. pen_stack/rag/llm.py +215 -0
  138. pen_stack/rag/qa.py +105 -0
  139. pen_stack/rag/retrieve.py +48 -0
  140. pen_stack/rules/__init__.py +9 -0
  141. pen_stack/rules/evaluators.py +318 -0
  142. pen_stack/rules/loader.py +31 -0
  143. pen_stack/rules/schema.py +99 -0
  144. pen_stack/rules/solver.py +43 -0
  145. pen_stack/rules/spec.py +78 -0
  146. pen_stack/safety/__init__.py +21 -0
  147. pen_stack/safety/audit.py +90 -0
  148. pen_stack/safety/gate.py +58 -0
  149. pen_stack/safety/pfam_scan.py +157 -0
  150. pen_stack/safety/policy.py +69 -0
  151. pen_stack/safety/redteam.py +71 -0
  152. pen_stack/safety/registry.py +255 -0
  153. pen_stack/safety/screen.py +59 -0
  154. pen_stack/safety/standards.py +141 -0
  155. pen_stack/score/__init__.py +1 -0
  156. pen_stack/score/recalibrate.py +77 -0
  157. pen_stack/score/therapeutic.py +85 -0
  158. pen_stack/server/__init__.py +1 -0
  159. pen_stack/server/api.py +647 -0
  160. pen_stack/spec/__init__.py +18 -0
  161. pen_stack/spec/clarify.py +42 -0
  162. pen_stack/spec/extract.py +406 -0
  163. pen_stack/spec/resolvers/__init__.py +17 -0
  164. pen_stack/spec/resolvers/cell.py +51 -0
  165. pen_stack/spec/resolvers/chem.py +32 -0
  166. pen_stack/spec/resolvers/feature.py +36 -0
  167. pen_stack/spec/resolvers/gene.py +43 -0
  168. pen_stack/spec/resolvers/locus.py +29 -0
  169. pen_stack/spec/resolvers/phenotype.py +37 -0
  170. pen_stack/spec/satisfy.py +114 -0
  171. pen_stack/spec/service.py +33 -0
  172. pen_stack/spec/writespec.py +252 -0
  173. pen_stack/twin/__init__.py +14 -0
  174. pen_stack/twin/calibrate.py +61 -0
  175. pen_stack/twin/data/__init__.py +12 -0
  176. pen_stack/twin/data/position_effect.py +245 -0
  177. pen_stack/twin/mechanistic.py +147 -0
  178. pen_stack/twin/outcome.py +132 -0
  179. pen_stack/twin/position_effect.py +454 -0
  180. pen_stack/ui/__init__.py +1 -0
  181. pen_stack/ui/app.py +713 -0
  182. pen_stack/validate/__init__.py +1 -0
  183. pen_stack/validate/adapt_demo.py +69 -0
  184. pen_stack/validate/agent_eval.py +117 -0
  185. pen_stack/validate/bench_adversarial_tasks.py +118 -0
  186. pen_stack/validate/bench_coscientist_tasks.py +60 -0
  187. pen_stack/validate/bench_graph_tasks.py +64 -0
  188. pen_stack/validate/bench_rule_tasks.py +84 -0
  189. pen_stack/validate/bench_trust_tasks.py +92 -0
  190. pen_stack/validate/bench_writetype_tasks.py +101 -0
  191. pen_stack/validate/blind_gsh_discovery.py +261 -0
  192. pen_stack/validate/cargo_directionality.py +57 -0
  193. pen_stack/validate/closed_loop.py +63 -0
  194. pen_stack/validate/durability_baselines.py +185 -0
  195. pen_stack/validate/experiment_design.py +65 -0
  196. pen_stack/validate/expr_controls.py +39 -0
  197. pen_stack/validate/forward_hypotheses.py +104 -0
  198. pen_stack/validate/generative_design.py +62 -0
  199. pen_stack/validate/guide_qc_demo.py +69 -0
  200. pen_stack/validate/heldout_celltype_expr.py +32 -0
  201. pen_stack/validate/immune_calibration.py +133 -0
  202. pen_stack/validate/intent_specification.py +82 -0
  203. pen_stack/validate/known_biology_expr.py +38 -0
  204. pen_stack/validate/offtarget_energetics_eval.py +144 -0
  205. pen_stack/validate/out_of_scope_refusal.py +82 -0
  206. pen_stack/validate/outcome_calibration.py +194 -0
  207. pen_stack/validate/outcome_prediction.py +76 -0
  208. pen_stack/validate/paper3_benchmark.py +165 -0
  209. pen_stack/validate/paper4_real_validation.py +144 -0
  210. pen_stack/validate/paper4_validation.py +82 -0
  211. pen_stack/validate/protocol_safety.py +62 -0
  212. pen_stack/validate/safety_screening.py +72 -0
  213. pen_stack/validate/selective_prediction.py +104 -0
  214. pen_stack/validate/seq_vs_measured.py +134 -0
  215. pen_stack/validate/target_site_controls.py +65 -0
  216. pen_stack/validate/uncertainty_eval.py +244 -0
  217. pen_stack/validate/ungrounded_baseline.py +234 -0
  218. pen_stack/validate/within_locus_ranking.py +84 -0
  219. pen_stack/validate/writer_recovery.py +91 -0
  220. pen_stack/verify/__init__.py +5 -0
  221. pen_stack/verify/proof.py +206 -0
  222. pen_stack/verify/schema.py +53 -0
  223. pen_stack/verify/service.py +191 -0
  224. pen_stack/web/__init__.py +18 -0
  225. pen_stack/web/guide.py +110 -0
  226. pen_stack/web/llm.py +393 -0
  227. pen_stack/web/llm_provider.py +119 -0
  228. pen_stack/web/router.py +119 -0
  229. pen_stack/web/server.py +96 -0
  230. pen_stack/web/tools.py +197 -0
  231. pen_stack/wgenome/__init__.py +1 -0
  232. pen_stack/wgenome/chromatin_seq.py +83 -0
  233. pen_stack/wgenome/durability.py +108 -0
  234. pen_stack/wgenome/export_tracks.py +52 -0
  235. pen_stack/wgenome/features.py +82 -0
  236. pen_stack/wgenome/genotoxic_blocklist.py +88 -0
  237. pen_stack/wgenome/gsh_baseline.py +154 -0
  238. pen_stack/wgenome/mesh_features.py +61 -0
  239. pen_stack/wgenome/offtarget_assay.py +80 -0
  240. pen_stack/wgenome/offtarget_bridge.py +47 -0
  241. pen_stack/wgenome/offtarget_cast.py +97 -0
  242. pen_stack/wgenome/offtarget_data.py +148 -0
  243. pen_stack/wgenome/offtarget_enumerate.py +274 -0
  244. pen_stack/wgenome/offtarget_integrase.py +155 -0
  245. pen_stack/wgenome/offtarget_nuclease.py +123 -0
  246. pen_stack/wgenome/offtarget_paste.py +41 -0
  247. pen_stack/wgenome/offtarget_predict.py +282 -0
  248. pen_stack/wgenome/ood.py +135 -0
  249. pen_stack/wgenome/providers.py +278 -0
  250. pen_stack/wgenome/safety.py +69 -0
  251. pen_stack/wgenome/structure3d.py +212 -0
  252. pen_stack/wgenome/uncertainty.py +250 -0
  253. pen_stack/wgenome/writability.py +72 -0
  254. pen_stack-0.1.0.dist-info/METADATA +401 -0
  255. pen_stack-0.1.0.dist-info/RECORD +259 -0
  256. pen_stack-0.1.0.dist-info/WHEEL +5 -0
  257. pen_stack-0.1.0.dist-info/entry_points.txt +3 -0
  258. pen_stack-0.1.0.dist-info/licenses/LICENSE +21 -0
  259. pen_stack-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,245 @@
1
+ """Position-effect / expression supervision, the unified table.
2
+
3
+ ONE schema over the scattered human/mouse position-effect datasets, so a single learned cassette x context
4
+ model (`twin.position_effect.PositionEffectModel`) can be trained and a held-out-cell-type benchmark
5
+ (`benchmarks/position_effect/`, TPE-Bench) can be sealed. Wrap, do not rebuild: TRIP supervision is the same
6
+ table the durability head already uses; this module just unifies it with the other sources and adds the
7
+ cassette identity + cross-dataset normalization + leakage-controlled splits.
8
+
9
+ NO FABRICATION. Each dataset is registered with its verified accession/DOI and a loader. A dataset whose raw
10
+ data is not present is reported `available=False` (an "not fetched"), never silently imputed. The only
11
+ dataset wired live is **TRIP** (Akhtar 2013, on the VM + pulled locally); the additional human
12
+ position-effect sources (PatchMPRA, MPIRE, lentiMPRA, Leemans) are registered with their accessions and a
13
+ loader contract, and become available when their raw data is fetched, the cross-cell-type *transfer* claim is
14
+ explicitly gated on that acquisition (documented, not asserted).
15
+
16
+ Canonical schema (one row = one integrated reporter / element measurement):
17
+ dataset, organism, cell_type, chrom, pos, cassette, expression_raw, expression_z, silenced, <features...>
18
+ `expression_z` is z-scored WITHIN (dataset x cassette) so a promoter's strength does not leak across datasets
19
+ and the model learns the *context* effect on a comparable scale.
20
+ """
21
+ from __future__ import annotations
22
+
23
+ from dataclasses import dataclass
24
+ from pathlib import Path
25
+ from typing import Callable
26
+
27
+ import numpy as np
28
+ import pandas as pd
29
+
30
+ from pen_stack._resources import project_root
31
+
32
+ # canonical chromatin context features (subset present per dataset; missing -> excluded, never imputed silently)
33
+ FEATURE_COLS = ["atac", "dnase", "H3K27ac", "H3K4me1", "H3K4me3", "H3K9me3", "H3K27me3", "H3K36me3"]
34
+
35
+ SCHEMA = ["dataset", "organism", "cell_type", "chrom", "pos", "cassette",
36
+ "expression_raw", "expression_z", "silenced"]
37
+
38
+
39
+ # dataset registry with verified accessions/DOIs for each source dataset
40
+ @dataclass
41
+ class Dataset:
42
+ name: str
43
+ citation: str
44
+ doi: str
45
+ accession: str
46
+ organism: str
47
+ cell_types: tuple[str, ...]
48
+ role: str # "position_effect" (locus->expr) | "cassette_activity" (CRE/MPRA)
49
+ loader: Callable[[Path], pd.DataFrame] | None = None
50
+ note: str = ""
51
+ _rel: str = "" # repo-relative raw path the loader reads (for availability check)
52
+
53
+ def available(self, root: Path | None = None) -> bool:
54
+ if self.loader is None or not self._rel:
55
+ return False
56
+ root = root or project_root()
57
+ return (root / self._rel).exists()
58
+
59
+
60
+ def _load_trip(root: Path) -> pd.DataFrame:
61
+ """TRIP (Akhtar 2013) mESC integrations + mES chromatin marks -> canonical schema.
62
+
63
+ The parquet is the same `trip_with_chromatin.parquet` the durability head trains on (chrom, pos, promoter,
64
+ expression [log2], silenced, + 5 histone marks). cassette = promoter; cell_type = mESC.
65
+ """
66
+ path = root / "data/external/trip/trip_with_chromatin.parquet"
67
+ df = pd.read_parquet(path)
68
+ marks = [c for c in FEATURE_COLS if c in df.columns]
69
+ out = pd.DataFrame({
70
+ "dataset": "TRIP_Akhtar2013",
71
+ "organism": "mouse",
72
+ "cell_type": "mESC",
73
+ "chrom": df["chrom"].astype(str),
74
+ "pos": df["pos"].astype("int64"),
75
+ "cassette": df["promoter"].astype(str),
76
+ "expression_raw": df["expression"].astype(float),
77
+ "silenced": df["silenced"].astype(bool),
78
+ })
79
+ for m in marks:
80
+ out[m] = df[m].astype(float)
81
+ return out
82
+
83
+
84
+ def _load_leemans(root: Path) -> pd.DataFrame:
85
+ """Leemans 2019 human K562 TRIP integrations + K562 chromatin marks -> canonical schema.
86
+
87
+ The parquet (chrom, pos, promoter, expression [log], silenced, + chromatin marks present in K562) is the
88
+ fetched Leemans position-effect table: per-barcode coordinates from the Cell supplement Dataset S2 joined
89
+ to the van Steensel feature set. cassette = promoter; cell_type = K562. This is the human-cell-type
90
+ supervision that flips the cross-cell-type transfer axis from data-gated to fetched.
91
+ """
92
+ path = root / "data/external/leemans/leemans_k562_position_effect.parquet"
93
+ df = pd.read_parquet(path)
94
+ marks = [c for c in FEATURE_COLS if c in df.columns]
95
+ out = pd.DataFrame({
96
+ "dataset": "Leemans2019",
97
+ "organism": "human",
98
+ "cell_type": "K562",
99
+ "chrom": df["chrom"].astype(str),
100
+ "pos": df["pos"].astype("int64"),
101
+ "cassette": df["promoter"].astype(str),
102
+ "expression_raw": df["expression"].astype(float),
103
+ "silenced": df["silenced"].astype(bool),
104
+ })
105
+ for m in marks:
106
+ out[m] = df[m].astype(float)
107
+ return out
108
+
109
+
110
+ def _loader_not_fetched(name: str, accession: str) -> Callable[[Path], pd.DataFrame]:
111
+ def _raise(_root: Path) -> pd.DataFrame:
112
+ raise FileNotFoundError(
113
+ f"{name}: raw data not fetched (accession {accession}). Wires TRIP only; this source is "
114
+ f"registered with its accession + loader contract and becomes available once fetched. The "
115
+ f"cross-cell-type transfer claim is gated on this acquisition (not asserted).")
116
+ return _raise
117
+
118
+
119
+ DATASETS: dict[str, Dataset] = {
120
+ "TRIP_Akhtar2013": Dataset(
121
+ name="TRIP_Akhtar2013",
122
+ citation="Akhtar et al., Cell 2013 (thousands of reporters integrated in parallel)",
123
+ doi="10.1016/j.cell.2013.07.018",
124
+ accession="GEO GSE49806 (tetO) + GSE49807 (mPGK); trip.nki.nl",
125
+ organism="mouse", cell_types=("mESC",), role="position_effect",
126
+ loader=_load_trip, _rel="data/external/trip/trip_with_chromatin.parquet",
127
+ note="position effect on an integrated cassette, the writing-relevant supervision (LIVE)."),
128
+ "PatchMPRA_Maricque2019": Dataset(
129
+ name="PatchMPRA_Maricque2019",
130
+ citation="Maricque, Chaudhari & Cohen, Nat Biotechnol 2019 (genomically integrated MPRA)",
131
+ doi="10.1038/nbt.4285", accession="GEO (per paper)",
132
+ organism="mouse", cell_types=("mESC",), role="position_effect",
133
+ loader=_loader_not_fetched("PatchMPRA_Maricque2019", "10.1038/nbt.4285"),
134
+ note="cassette x context separability evidence (data-gated)."),
135
+ "MPIRE_Hong2024": Dataset(
136
+ name="MPIRE_Hong2024",
137
+ citation="Hong et al., Nat Commun 2024 (massively parallel insulator-activity MPRA)",
138
+ doi="10.1038/s41467-024-52599-6", accession="GEO GSE223403; github.com/claricehong/MPIRE_insulators",
139
+ organism="human", cell_types=("K562",), role="position_effect",
140
+ loader=_loader_not_fetched("MPIRE_Hong2024", "GSE223403"),
141
+ note="human K562 position/insulator context (data-gated)."),
142
+ "lentiMPRA_Agarwal2025": Dataset(
143
+ name="lentiMPRA_Agarwal2025",
144
+ citation="Agarwal et al., Nature 639:411-420 (2025) (~680k regulatory elements)",
145
+ doi="10.1038/s41586-024-08430-9", accession="ENCODE portal + GEO; bioRxiv 2023.03.05.531189",
146
+ organism="human", cell_types=("HepG2", "K562", "WTC11"), role="cassette_activity",
147
+ loader=_loader_not_fetched("lentiMPRA_Agarwal2025", "10.1038/s41586-024-08430-9"),
148
+ note="cassette/CRE activity supervision across 3 human cell types (data-gated)."),
149
+ "Leemans2019": Dataset(
150
+ name="Leemans2019",
151
+ citation="Leemans et al., Cell 2019 (promoter-intrinsic + local chromatin determine repression in LADs)",
152
+ doi="10.1016/j.cell.2019.03.009", accession="GEO (per paper); van Steensel lab",
153
+ organism="human", cell_types=("K562",), role="position_effect",
154
+ loader=_load_leemans, _rel="data/external/leemans/leemans_k562_position_effect.parquet",
155
+ note="human K562 LAD-repression position effect (LIVE: fetched via P2 -- coordinates from the "
156
+ "Cell supplement Dataset S2 joined to the van Steensel feature set)."),
157
+ }
158
+
159
+
160
+ def available_datasets(root: Path | None = None) -> list[str]:
161
+ root = root or project_root()
162
+ return [k for k, d in DATASETS.items() if d.available(root)]
163
+
164
+
165
+ # normalization + loader
166
+ def normalize_within(df: pd.DataFrame, by=("dataset", "cassette"), col: str = "expression_raw",
167
+ out: str = "expression_z") -> pd.DataFrame:
168
+ """z-score expression within each (dataset x cassette) group, so a cassette's intrinsic strength does not
169
+ leak across datasets and the model is supervised on the *context* deviation on a comparable scale. A
170
+ singleton/zero-variance group maps to 0.0 (no spurious scale)."""
171
+ df = df.copy()
172
+ def _z(s: pd.Series) -> pd.Series:
173
+ sd = s.std(ddof=0)
174
+ return (s - s.mean()) / sd if sd and sd > 1e-12 else pd.Series(0.0, index=s.index)
175
+ df[out] = df.groupby(list(by))[col].transform(_z)
176
+ return df
177
+
178
+
179
+ def load_position_effect(datasets: list[str] | None = None, root: Path | None = None,
180
+ require: bool = False) -> pd.DataFrame:
181
+ """Unified position-effect table. Loads every AVAILABLE registered dataset (or the named subset),
182
+ concatenates to the canonical schema, and z-normalizes expression within (dataset x cassette).
183
+
184
+ `require=True` raises if a requested dataset is unavailable; default skips unavailable ones (logging the
185
+ skip in the returned frame's `.attrs['skipped']`), clear about what is and is not in the table.
186
+ """
187
+ root = root or project_root()
188
+ names = datasets or list(DATASETS)
189
+ frames, skipped = [], []
190
+ for name in names:
191
+ d = DATASETS[name]
192
+ if not d.available(root):
193
+ skipped.append(name)
194
+ if require:
195
+ d.loader(root) # raise the informative FileNotFoundError
196
+ continue
197
+ frames.append(d.loader(root))
198
+ if not frames:
199
+ raise FileNotFoundError(
200
+ f"no position-effect dataset available under {root}. Available registry: {list(DATASETS)}; "
201
+ f"skipped (not fetched): {skipped}. Pull TRIP via scratch/v67_pull_trip.py or set PEN_STACK_HOME.")
202
+ df = pd.concat(frames, ignore_index=True)
203
+ df = normalize_within(df)
204
+ df.attrs["skipped"] = skipped
205
+ df.attrs["datasets"] = [f["dataset"].iloc[0] for f in frames]
206
+ # reorder: schema first, then whatever feature columns are present
207
+ feats = [c for c in FEATURE_COLS if c in df.columns]
208
+ return df[[c for c in SCHEMA if c in df.columns] + feats]
209
+
210
+
211
+ # leakage-controlled splits
212
+ def blocked_splits(df: pd.DataFrame, n_splits: int = 5, group: str = "chrom",
213
+ seed: int = 20260619) -> list[tuple[np.ndarray, np.ndarray]]:
214
+ """Domain-blocked CV: no `group` (default chromosome) value appears in both train and test of a fold,
215
+ the leakage control the position-effect task needs (nearby integrations share chromatin)."""
216
+ from sklearn.model_selection import GroupKFold
217
+ g = df[group].astype("category").cat.codes.to_numpy()
218
+ k = min(n_splits, len(np.unique(g)))
219
+ return list(GroupKFold(n_splits=k).split(df, groups=g))
220
+
221
+
222
+ def heldout_celltype_splits(df: pd.DataFrame) -> list[tuple[str, np.ndarray, np.ndarray]]:
223
+ """Leave-one-cell-type-out: train on all but one cell type, test on the held-out one. This is the headline
224
+ transfer evaluation. With a single available cell type it returns [] and the caller reports the transfer
225
+ axis as data-gated, never a fabricated transfer number."""
226
+ cts = sorted(df["cell_type"].unique())
227
+ if len(cts) < 2:
228
+ return []
229
+ out = []
230
+ idx = np.arange(len(df))
231
+ for ct in cts:
232
+ te = idx[df["cell_type"].to_numpy() == ct]
233
+ tr = idx[df["cell_type"].to_numpy() != ct]
234
+ out.append((ct, tr, te))
235
+ return out
236
+
237
+
238
+ def leakage_report(df: pd.DataFrame, splits: list[tuple[np.ndarray, np.ndarray]],
239
+ group: str = "chrom") -> dict:
240
+ """Verify no `group` value is co-located across the train/test of any fold (the split's integrity claim)."""
241
+ bad = 0
242
+ for tr, te in splits:
243
+ if set(df.iloc[tr][group]) & set(df.iloc[te][group]):
244
+ bad += 1
245
+ return {"n_folds": len(splits), "folds_with_leakage": bad, "clean": bad == 0, "group": group}
@@ -0,0 +1,147 @@
1
+ """Mechanistic simulation where computable.
2
+
3
+ Computes the consequences MECHANISM allows, steady-state cassette expression in a chromatin context, and
4
+ nothing more. Physics where computable; explicitly NOT a phenotype, NOT in-vivo behaviour, NOT durability beyond
5
+ the steady state. Assumptions and scope flags travel with every output.
6
+
7
+ The promoter palette is a comprehensive, literature-cited table (configs/expression/promoters.yaml,
8
+ ~25 constitutive + tissue-specific promoters) and the cassette carries an MODIFIER profile (WPRE, intron,
9
+ polyA, Kozak, codon-optimization, CpG-silencing, configs/expression/modifiers.yaml). Modifier effects are
10
+ transgene-dependent PRIORS, so they are reported as a bounded uplift RANGE with wide uncertainty, never folded
11
+ into the [0,1] base score as a point estimate. Absolute titer / % of normal stays a known-unknown.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ from functools import lru_cache
16
+
17
+ import yaml
18
+
19
+ from pen_stack._resources import resource
20
+
21
+ # legacy default keys kept for backward compatibility (earlier designs used these names directly).
22
+ _LEGACY_STRENGTH = {"ef1a": 1.0, "cag": 1.0, "cmv": 0.9, "pgk": 0.6, "ubc": 0.5,
23
+ "endogenous": 0.4, "minimal": 0.2}
24
+
25
+
26
+ @lru_cache(maxsize=1)
27
+ def _palette() -> dict:
28
+ """Flatten the comprehensive promoter palette (constitutive + tissue-specific) to {name: entry}."""
29
+ try:
30
+ cfg = yaml.safe_load(resource("configs/expression/promoters.yaml").read_text(encoding="utf-8"))
31
+ except Exception: # noqa: BLE001 - config absent (shouldn't happen) -> legacy table only
32
+ return {}
33
+ out: dict = {}
34
+ for group in ("constitutive", "tissue_specific"):
35
+ for name, e in (cfg.get(group) or {}).items():
36
+ out[name] = e
37
+ out["_default"] = cfg.get("default_strength", 0.5)
38
+ return out
39
+
40
+
41
+ @lru_cache(maxsize=1)
42
+ def _modifiers() -> dict:
43
+ try:
44
+ return yaml.safe_load(resource("configs/expression/modifiers.yaml").read_text(encoding="utf-8"))
45
+ except Exception: # noqa: BLE001
46
+ return {"modifiers": {}, "composition": {"max_total_uplift": 4.0}}
47
+
48
+
49
+ def promoter_palette() -> list[dict]:
50
+ """The selectable promoter palette (constitutive + tissue-specific), each with its literature strength,
51
+ context and citation. Surfaced so the Twin can offer a promoter selector, the mechanistic relative-expression
52
+ estimate reacts to the chosen promoter (it defaults to 0.5 when none is picked)."""
53
+ try:
54
+ cfg = yaml.safe_load(resource("configs/expression/promoters.yaml").read_text(encoding="utf-8"))
55
+ except Exception: # noqa: BLE001
56
+ return []
57
+ out: list[dict] = []
58
+ for group in ("constitutive", "tissue_specific"):
59
+ for name, e in (cfg.get(group) or {}).items():
60
+ if isinstance(e, dict):
61
+ out.append({"name": name, "strength": e.get("strength"), "context": e.get("context"),
62
+ "citation": e.get("citation"), "group": group})
63
+ return out
64
+
65
+
66
+ def promoter_info(name: str) -> dict:
67
+ """The cited palette entry for a promoter (strength + context + assay + citation), or a labelled default.
68
+
69
+ A name the palette does not carry is never scored as though it were a real promoter: the returned entry marks
70
+ ``recognized=False`` and echoes the requested name, so a typo is distinguishable from choosing no promoter at
71
+ all (both fall back to the neutral default strength, but only one is a caller mistake)."""
72
+ pal = _palette()
73
+ key = str(name or "").strip().lower()
74
+ if key in pal:
75
+ return {**pal[key], "recognized": True}
76
+ if key in _LEGACY_STRENGTH:
77
+ return {"strength": _LEGACY_STRENGTH[key], "context": "ubiquitous", "confidence": "legacy",
78
+ "recognized": True}
79
+ return {"strength": pal.get("_default", 0.5), "context": "unknown", "confidence": "default",
80
+ "recognized": False, "requested_promoter": (str(name).strip() or None) if name else None}
81
+
82
+
83
+ def _promoter_strength(design: dict) -> tuple[float, dict]:
84
+ p = design.get("promoter")
85
+ if isinstance(p, dict) and p.get("strength") is not None:
86
+ return float(p["strength"]), {"context": p.get("context", "supplied"), "confidence": "supplied"}
87
+ name = p if isinstance(p, str) else str(p or "")
88
+ info = promoter_info(name)
89
+ return float(info.get("strength", 0.5)), info
90
+
91
+
92
+ def _modifier_profile(design: dict) -> dict:
93
+ """A bounded modifier uplift RANGE from the cassette's cis-elements. NOT a point multiplier and NOT
94
+ folded into the [0,1] base, modifier effects are transgene/context-dependent priors with wide uncertainty."""
95
+ mods = _modifiers().get("modifiers", {})
96
+ cap = float(_modifiers().get("composition", {}).get("max_total_uplift", 4.0))
97
+ applied, hi = [], 1.0 # lower bound stays 1.0: a modifier may do nothing for THIS transgene
98
+ for key, present in (("wpre", design.get("wpre")), ("intron", design.get("intron"))):
99
+ if present and key in mods:
100
+ hi *= float(mods[key].get("present_fold", [1.0, 1.0])[1])
101
+ applied.append(key)
102
+ if design.get("codon_optimized") and "codon_optimization" in mods:
103
+ hi *= float(mods["codon_optimization"].get("optimized_fold", [1.0, 1.0])[1])
104
+ applied.append("codon_optimization")
105
+ cpg_flag = bool(design.get("high_cpg") or (design.get("cpg_oe") is not None and float(design.get("cpg_oe")) > 0.6))
106
+ if cpg_flag:
107
+ applied.append("cpg_silencing(down)")
108
+ lo, hi = 1.0, round(min(hi, cap), 2)
109
+ return {
110
+ "applied": applied,
111
+ "estimated_uplift_range": [round(lo, 2), round(hi, 2)],
112
+ "direction_note": ("net uplift expected (transgene-dependent)" if hi > 1.0 else "no modifier signal"),
113
+ "cpg_silencing_risk": bool(cpg_flag),
114
+ "caveat": "modifier effects are transgene/cell/construct-dependent PRIORS (e.g. an intron is ~20x for one "
115
+ "transgene, ~3x for another), a bounded range, not a measured multiplier for THIS construct.",
116
+ "source": "configs/expression/modifiers.yaml (literature priors)",
117
+ }
118
+
119
+
120
+ def cassette_expression(design: dict, chromatin_ctx: dict) -> dict:
121
+ """Computable steady-state RELATIVE expression from an integrated cassette: promoter_strength x copy_number x
122
+ accessibility (base, [0,1]-relative), PLUS an bounded modifier uplift range. NOT a phenotype, NOT an
123
+ absolute titer."""
124
+ p, pinfo = _promoter_strength(design)
125
+ cn = float(design.get("copy_number", 1) or 1)
126
+ acc = float(chromatin_ctx.get("accessibility", 1.0) if chromatin_ctx else 1.0)
127
+ rel = p * cn * acc
128
+ flags = ["episomal_durability_unknown", "phenotype_not_modeled"]
129
+ # context scope: a promoter that silences (CMV, SFFV in stem cells) is flagged, not silently trusted.
130
+ if str(pinfo.get("context", "")).startswith("variable") or "SILENCE" in str(pinfo.get("note", "")).upper():
131
+ flags.append("promoter_context_variable_or_silencing")
132
+ # a named promoter the palette does not carry falls back to the neutral default: say so rather than let a typo
133
+ # read as a scored, cited promoter.
134
+ if pinfo.get("recognized") is False and pinfo.get("requested_promoter"):
135
+ flags.append("promoter_not_in_palette_neutral_default_used")
136
+ return {
137
+ "relative_expression": round(rel, 4),
138
+ "units": "relative (dimensionless)",
139
+ "modifier_profile": _modifier_profile(design),
140
+ "assumptions": ["steady-state", "no silencing dynamics modeled", "linear copy scaling",
141
+ "promoter strength is an ordinal literature prior (context-dependent)"],
142
+ "scope_flags": flags,
143
+ "provenance": {"promoter_strength": p, "promoter_context": pinfo.get("context"),
144
+ "promoter_recognized": pinfo.get("recognized", True),
145
+ "promoter_citation": pinfo.get("citation"), "copy_number": cn, "accessibility": acc,
146
+ "source": "twin.mechanistic (closed-form steady state + literature promoter palette)"},
147
+ }
@@ -0,0 +1,132 @@
1
+ """Write-outcome prediction, the digital twin.
2
+
3
+ Fuses what MECHANISM computes (cassette expression), what an IN-DISTRIBUTION virtual-cell model supports
4
+ (deferred/cache-replayed, OOD-gated), and the IMMUNE profile (sourced, never invented) into a single
5
+ prediction with an interval that WIDENS under OOD and an explicit boundary at phenotype. The twin is a
6
+ hypothesis engine: every output is a CANDIDATE with an interval; phenotype, in-vivo behaviour, immunogenicity
7
+ magnitude, and durability beyond the computable stay scope-flagged.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ _BASE_HALF_WIDTH = 0.20 # heuristic band on the fused estimate (NOT a trained conformal interval)
12
+ _OOD_INFLATION = 1.6 # OOD widens the interval rather than over-trusting an extrapolating model
13
+
14
+
15
+ def _chromatin(cell_state: str, design: dict) -> dict:
16
+ """Chromatin context for the mechanistic model. Uses a supplied accessibility if present, else neutral 1.0
17
+ (and flags that a full epigenome was not supplied)."""
18
+ acc = design.get("accessibility")
19
+ if acc is None and isinstance(design.get("chromatin_ctx"), dict):
20
+ acc = design["chromatin_ctx"].get("accessibility")
21
+ return {"accessibility": float(acc) if acc is not None else 1.0,
22
+ "accessibility_supplied": acc is not None}
23
+
24
+
25
+ def _as_perturbation(design: dict) -> dict:
26
+ return {"kind": design.get("perturbation_kind") or "genetic",
27
+ "target": design.get("gene"), "write_type": design.get("write_type")}
28
+
29
+
30
+ def _in_vivo(design: dict) -> bool:
31
+ veh = design.get("delivery_vehicle")
32
+ if not veh:
33
+ return False
34
+ from pen_stack.planner.delivery_vehicles import vehicle as _veh
35
+ return bool((_veh(veh) or {}).get("in_vivo"))
36
+
37
+
38
+ def predict_outcome(design: dict, cell_state: str) -> dict:
39
+ from pen_stack.oracles.vcell import predict_response
40
+ from pen_stack.twin.mechanistic import cassette_expression
41
+ from pen_stack.twin.position_effect import predict_stage_h
42
+
43
+ mech = cassette_expression(design, _chromatin(cell_state, design))
44
+ # When a chromatin CONTEXT is supplied and the trained artifact is present, the learned, TRAINED-CONFORMAL
45
+ # position-effect model serves (interval + p_silenced + OOD widening). Without a
46
+ # context (or the artifact) this is None and the closed-form heuristic below stands, backward compatible.
47
+ learned = predict_stage_h(design, cell_type=cell_state)
48
+ vc = predict_response(cell_state, _as_perturbation(design), model="state")
49
+ imm = None
50
+ if design.get("delivery_vehicle"):
51
+ from pen_stack.planner.immune_profile import immune_profile
52
+ imm = immune_profile(design)
53
+
54
+ # fuse: the computable mechanistic estimate is the backbone; the VC model adds an in-distribution response
55
+ # estimate ONLY when available (deferred backend -> None, never fabricated).
56
+ est = mech["relative_expression"]
57
+ vc_value = vc.value if (vc.available and not vc.extrapolating and vc.value) else None
58
+
59
+ scope = list(mech["scope_flags"]) + ["in_vivo_magnitude_unknown"]
60
+ if vc.extrapolating:
61
+ scope.append("vcell_OOD")
62
+
63
+ # in-vivo durability MAY be conditioned on pre-existing NAb (the GROUNDED immune axis; no invented numbers)
64
+ conditioned_note = None
65
+ if imm is not None and _in_vivo(design):
66
+ nab = (imm.get("axes", {}).get("preexisting_nab") or {})
67
+ if nab.get("in_scope") and nab.get("value") is not None:
68
+ est = round(est * float(nab["value"]), 4) # higher NAb score = less pre-existing immunity
69
+ conditioned_note = f"in-vivo durability conditioned on grounded pre-existing NAb axis ({nab['value']})"
70
+
71
+ half = _BASE_HALF_WIDTH * (_OOD_INFLATION if vc.extrapolating else 1.0)
72
+ lo, hi = round(max(0.0, est - half), 4), round(est + half, 4)
73
+
74
+ # cargo-capacity feasibility: relative expression is PER-COPY and independent of cargo size (correct science),
75
+ # but a cargo that overflows the vehicle cannot be delivered as specified, surface it so a (say) 50 kb AAV
76
+ # cargo is not silently read as a buildable design. The capacity RULE + its repair live in Verify/legality.
77
+ feasibility = {"buildable": True}
78
+ veh = design.get("delivery_vehicle") or design.get("vehicle")
79
+ cargo_bp = design.get("cargo_bp")
80
+ if cargo_bp is not None and int(cargo_bp) <= 0:
81
+ # a non-positive cargo size is not a buildable construct, flag it rather than silently reporting
82
+ # buildable:True (tester finding: cargo_bp=0 was accepted as buildable).
83
+ feasibility = {"buildable": False, "cargo_bp": int(cargo_bp),
84
+ "reason": f"cargo size {int(cargo_bp)} bp is not a buildable construct (must be >= 1 bp)."}
85
+ scope.append("cargo_size_nonpositive")
86
+ elif veh and cargo_bp is not None:
87
+ from pen_stack.planner.delivery_vehicles import vehicle as _veh
88
+ cap = (_veh(veh) or {}).get("cargo_capacity_bp")
89
+ if cap is not None and int(cargo_bp) > int(cap):
90
+ feasibility = {"buildable": False, "cargo_bp": int(cargo_bp), "vehicle": veh, "capacity_bp": int(cap),
91
+ "reason": (f"cargo {int(cargo_bp)} bp exceeds {veh} capacity {int(cap)} bp, not "
92
+ "deliverable as specified. Relative expression is per-copy (independent of "
93
+ "cargo size), so this estimate is for a construct that cannot be built as-is; "
94
+ "see Verify / legality for the capacity rule and its repair.")}
95
+ scope.append("cargo_exceeds_vehicle_capacity")
96
+
97
+ predicted = {"relative_expression": est, "vcell_response": vc_value, "units": mech["units"]}
98
+ interval_kind = ("heuristic band (widens under OOD); NOT a trained conformal interval - no public "
99
+ "perturbation-outcome calibration set (Arc VCC: models do not yet beat naive baselines)")
100
+ if learned is not None:
101
+ # the learned model serves: surface its trained-conformal value + a relative rescale alongside the
102
+ # mechanistic estimate; the mechanistic `interval` is retained for the relative-scale contract.
103
+ predicted["relative_expression_learned"] = learned.get("relative_expression_learned")
104
+ predicted["learned_log2_expression"] = learned.get("predicted_log2_expression")
105
+ interval_kind = ("trained-conformal: see position_effect.interval_log2 (split-conformal on TRIP "
106
+ "supervision, OOD-widened). Mechanistic relative interval retained below.")
107
+ scope = scope + ["stage_h_learned_trained_conformal"]
108
+
109
+ return {
110
+ "predicted_outcome": predicted,
111
+ "interval": [lo, hi],
112
+ "interval_kind": interval_kind,
113
+ "feasibility": feasibility, # cargo-vs-vehicle capacity: is this construct buildable as specified?
114
+ # the mechanistic top-level is promoter x copy-number x accessibility x pre-existing-NAb; accessibility is
115
+ # neutral (1.0) here because no cell-type chromatin track is mounted, so it is deliberately CELL-TYPE-AGNOSTIC
116
+ #, the cell-type-specific position effect is the learned position-effect head (`position_effect`), not this number.
117
+ "relative_expression_basis": ("promoter strength x copy number x accessibility (neutral 1.0, cell-type "
118
+ "position effect is the learned position-effect head, not the mechanistic top-level)"
119
+ + (" x pre-existing-NAb" if conditioned_note else "")),
120
+ "position_effect": learned, # learned trained-conformal block (or None)
121
+ "stage_h_mode": "learned_trained_conformal" if learned is not None else "heuristic",
122
+ "immune_outcome": imm, # immune profile (or None if no vehicle)
123
+ "extrapolating": bool(vc.extrapolating),
124
+ "conditioned_on_preexisting_nab": conditioned_note,
125
+ "scope_flags": scope,
126
+ "provenance": {"mechanistic": mech["assumptions"], "vcell": vc.provenance.model_dump()
127
+ if hasattr(vc.provenance, "model_dump") else str(vc.provenance),
128
+ "immune": "profile" if imm is not None else None,
129
+ "position_effect": (learned["provenance"] if learned else None)},
130
+ "output_kind": "candidate",
131
+ "no_fabrication": True,
132
+ }