pen-stack 0.1.0__py3-none-any.whl

pen_stack/atlas/writer_efficiency.py

@@ -0,0 +1,184 @@
+"""Writer-Efficiency dataset, the curated, literature-sourced efficiency table.
+
+The writer frontier crossed into quantitative human-cell data in 2023-26, but the efficiency numbers are scattered
+across a dozen papers with no unified resource. This module curates them into ONE schema so a learned cross-family
+efficiency predictor (`atlas.writer_predict`) can be trained and a held-out benchmark (`benchmarks/writer_efficiency/`,
+the Writer-Efficiency Bench) can be sealed. The curated dataset is itself the standalone contribution.
+
+NO FABRICATION. Every record carries its DOI, the verbatim source quote, and a `source_access` provenance grade:
+  * `pmc_verbatim`, quoted from the open-access PMC full text (highest confidence)
+  * `abstract`, quoted from the published abstract
+  * `secondary`, a number reported by a reliable secondary source (press release / summary) where the
+                      primary full text is paywalled; flagged so it can be excluded from the strict benchmark.
+Range values (e.g. "~10-20%", "4-22%") are stored as their midpoint in `efficiency_pct` with the raw string in
+`efficiency_raw`; the exact quote travels in `quote`. Efficiencies are % integration in the stated cell type
+(human unless noted); bacterial (E. coli) rows are labelled and excluded from the human-cell benchmark.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+
+from pen_stack._resources import project_root
+
+COLUMNS = ["system", "family", "write_type", "variant", "cargo_bp", "locus", "cell_type", "organism",
+           "delivery", "efficiency_pct", "efficiency_raw", "specificity_pct", "source_access", "doi", "quote"]
+
+# Family keys align with the Writer Atlas (wtkb). PE-installed-att systems (PASTE/PASSIGE) -> PE_integrase;
+# pre-installed-att recombinase-only systems -> serine_integrase; bridge -> bridge_IS110; CAST -> CAST_VK.
+_R = [] # records
+
+
+def _rec(system, family, write_type, variant, cargo_bp, locus, cell_type, organism, delivery,
+         eff, eff_raw, doi, quote, source_access="pmc_verbatim", spec=None):
+    _R.append(dict(system=system, family=family, write_type=write_type, variant=variant, cargo_bp=cargo_bp,
+                   locus=locus, cell_type=cell_type, organism=organism, delivery=delivery,
+                   efficiency_pct=float(eff), efficiency_raw=eff_raw, specificity_pct=spec,
+                   source_access=source_access, doi=doi, quote=quote))
+
+
+# ---- PASTE, Yarnall et al., Nat Biotechnol 2023 (41:500-512), 10.1038/s41587-022-01527-4 (PMC10257351) ----
+_P = "10.1038/s41587-022-01527-4"
+_rec("PASTEv1", "PE_integrase", "insertion", "WT", 969, "ACTB", "HEK293FT", "human", "plasmid", 15, "~15%", _P,
+     "BxbINT integrase showed the highest integration rate (~15%) at the targeted ACTB locus")
+_rec("PASTEv1", "PE_integrase", "insertion", "WT", 969, "ACTB", "K562", "human", "plasmid", 15, "~15%", _P,
+     "PASTEv1 had ~15% gene integration activity in K562 cells")
+_rec("PASTEv1", "PE_integrase", "insertion", "WT", 969, "ACTB", "primary_T", "human", "plasmid", 5, "~5%", _P,
+     "around 5% efficiency in primary human T cells")
+_rec("PASTEv1", "PE_integrase", "insertion", "WT", 969, "ACTB", "primary_hepatocyte", "human", "plasmid", 5, "~5%",
+     _P, "PASTEv1 was capable of ~5% gene integration at the ACTB locus in non-dividing quiescent human primary hepatocytes")
+_rec("PASTEv3", "PE_integrase", "insertion", "WT", 36000, "ACTB", "HEK293FT", "human", "plasmid", 15, "~10-20%", _P,
+     "PASTEv3...achieved precise integration of templates as large as ~36,000 bp with ~10-20% integration efficiency at ACTB and LMNB1")
+_rec("PASTE", "PE_integrase", "insertion", "WT", 3000, "ACTB", "HEK293FT", "human", "plasmid", 13, "4-22%", _P,
+     "integration frequencies between 4% and 22% depending on the gene and insertion locus (cargos 969-4,906 bp)")
+_rec("PASTE_AdV", "PE_integrase", "insertion", "WT", 1000, "ACTB", "HepG2", "human", "adenovirus", 55, "~50-60%", _P,
+     "integration of up to ~50-60% with viral-only delivery in HEK293FT and HepG2 cells")
+_rec("PASTE_AdV", "PE_integrase", "insertion", "WT", 1000, "ACTB", "primary_hepatocyte", "human", "adenovirus",
+     10.5, "up to 10.5%", _P, "editing efficiencies up to 10.5% in a cargo-dependent fashion (primary human hepatocytes)")
+_rec("PASTE_invivo", "PE_integrase", "insertion", "WT", 1000, "ACTB", "hepatocyte_in_vivo", "human", "AAV", 2.5,
+     "up to 2.5%", _P, "PASTE was capable of integration rates as high as 2.5% in the human hepatocytes in the chimeric liver")
+
+# ---- (ee)PASSIGE, Pandey/Gao/Krasnow/Liu et al., Nat Biomed Eng 2025 (9:22-39), 10.1038/s41551-024-01227-1 (PMC11754103) ----
+_E = "10.1038/s41551-024-01227-1"
+_rec("eeBxb1", "serine_integrase", "insertion", "V105I", 5600, "AAVS1", "HEK293T", "human", "plasmid", 60, "60%", _E,
+     "The V105I mutant supported 60% and 39% integration efficiencies at AAVS1 and CCR5, respectively (pre-installed attB)")
+_rec("eeBxb1", "serine_integrase", "insertion", "V105I", 5600, "CCR5", "HEK293T", "human", "plasmid", 39, "39%", _E,
+     "The V105I mutant supported 60% and 39% integration efficiencies at AAVS1 and CCR5, respectively (pre-installed attB)")
+_rec("PASSIGE", "PE_integrase", "insertion", "WT_Bxb1", 5600, "AAVS1", "HEK293T", "human", "plasmid", 13, "13%", _E,
+     "evoPASSIGE: 27%, 22% vs. PASSIGE: 13%, 10% (at AAVS1 and CCR5)")
+_rec("PASSIGE", "PE_integrase", "insertion", "WT_Bxb1", 5600, "CCR5", "HEK293T", "human", "plasmid", 10, "10%", _E,
+     "evoPASSIGE: 27%, 22% vs. PASSIGE: 13%, 10% (at AAVS1 and CCR5)")
+_rec("evoPASSIGE", "PE_integrase", "insertion", "evoBxb1", 5600, "AAVS1", "HEK293T", "human", "plasmid", 27, "27%", _E,
+     "evoPASSIGE: 27%, 22% (at AAVS1 and CCR5)")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 5600, "AAVS1", "HEK293T", "human", "plasmid", 36, "36%", _E,
+     "36% and 27% at the AAVS1 and CCR5 loci, respectively")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 5600, "CCR5", "HEK293T", "human", "plasmid", 27, "27%", _E,
+     "36% and 27% at the AAVS1 and CCR5 loci, respectively")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 5600, "ROSA26", "N2a", "mouse", "plasmid", 20, "20%", _E,
+     "20% efficiency compared with 9.5% with evoPASSIGE and 3.2% with PASSIGE (Rosa26, N2a)")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 5600, "GBA1", "HEK293T", "human", "plasmid", 35, "35%", _E,
+     "therapeutic-gene integration; GBA1 among loci with 32% average and a minimum of 23% integration")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 5600, "FANCA", "HEK293T", "human", "plasmid", 46, "46%", _E,
+     "therapeutic-gene integration; FANCA among loci (32% average, minimum 23%)")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 5600, "B2M", "HEK293T", "human", "plasmid", 32, "32%", _E,
+     "32% average integration and a minimum of 23% integration (B2M)")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 5600, "ALB", "HEK293T", "human", "plasmid", 31, "31%", _E,
+     "therapeutic-gene integration; ALB ~31% (32% average, minimum 23%)")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 5800, "COL7A1", "primary_fibroblast", "human", "mRNA", 30,
+     "30%", _E, "eePASSIGE exhibited 30% and 18% average integration in the COL7A1 and FANCA loci, respectively (primary fibroblasts, mRNA)")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 5800, "FANCA", "primary_fibroblast", "human", "mRNA", 18,
+     "18%", _E, "eePASSIGE exhibited 30% and 18% average integration in the COL7A1 and FANCA loci, respectively (primary fibroblasts)")
+_rec("eePASSIGE", "PE_integrase", "insertion", "eeBxb1", 10500, "AAVS1", "HEK293T", "human", "plasmid", 35,
+     "up to 35%", _E, "eePASSIGE achieved up to 35% targeted integration of the 10.5-kb donor plasmid")
+_rec("eeBxb1", "serine_integrase", "insertion", "eeBxb1", 5600, "CCR5", "iPSC", "human", "plasmid", 3.8, "3.8%", _E,
+     "7.3-fold improvement in integration efficiency with eeBxb1 (3.8%) (human iPS cells)")
+_rec("PASTE", "PE_integrase", "insertion", "WT_Bxb1", 5600, "8loci_avg", "HEK293T", "human", "plasmid", 3.8,
+     "3.8% avg", _E, "eePASSIGE, evoPASSIGE, PASSIGE and PASTE mediated...average efficiency of 22%, 17%, 7.8% and 3.8%")
+
+# ---- Hyperactive integrases, Hew et al., Nucleic Acids Res 2024 (52(14):e64), 10.1093/nar/gkae534 (PMC) ----
+_H = "10.1093/nar/gkae534"
+_rec("Bxb1", "serine_integrase", "insertion", "WT", 6600, "Xq22.1", "K562", "human", "plasmid", 2.7, "2.7%", _H,
+     "11.2-fold (2.7% versus 30.3%) (WT versus evolved combination, PE-installed att)")
+_rec("Bxb1", "serine_integrase", "insertion", "c22_I87L_H95Y_V122M_A369P_E434G", 6600, "Xq22.1", "K562", "human",
+     "plasmid", 30.3, "30.3%", _H, "11.2-fold (2.7% versus 30.3%) (evolved combination mutant)")
+_rec("Bxb1", "serine_integrase", "insertion", "WT", 15700, "attP_preinstalled", "HEK293T", "human", "plasmid",
+     22.8, "22.8%", _H, "56.2% integration by the evolved Bxb1 integrase compared to 22.8% for wildtype (15.7 kb vWF cargo)")
+_rec("Bxb1", "serine_integrase", "insertion", "evolved_combination", 15700, "attP_preinstalled", "HEK293T", "human",
+     "plasmid", 56.2, "56.2%", _H, "56.2% integration by the evolved Bxb1 integrase (15.7 kb vWF cargo)")
+_rec("PhiC31", "serine_integrase", "insertion", "P2", 6600, "ROSA26", "HEK293T", "human", "plasmid", 1.3, "1.3%", _H,
+     "greatest improvement was between P2 (1.3%) and a 36-hour evolved variant P2-L2-1 (12.1%)")
+_rec("PhiC31", "serine_integrase", "insertion", "P2-L2-1", 6600, "ROSA26", "HEK293T", "human", "plasmid", 12.1,
+     "12.1%", _H, "P2 (1.3%) and a 36-hour evolved variant P2-L2-1 (12.1%)")
+_rec("PhiC31", "serine_integrase", "insertion", "P3-L1-2", 6600, "ROSA26", "HEK293T", "human", "plasmid", 18.4,
+     "18.4%", _H, "P3-L1-2 (18.4% at ROSA26)")
+_rec("PhiC31", "serine_integrase", "insertion", "P3-L1-13", 6600, "AAVS1", "HEK293T", "human", "plasmid", 11.1,
+     "11.1%", _H, "integration efficiencies for P3 reaching 11.1% and 3.0% at ROSA26 and AAVS1")
+
+# ---- CAST, ShCAST (Strecker 2019, 10.1126/science.aax9181, E. coli) + evoCAST (Science 2025, 10.1126/science.adt5199, PMC12326709) ----
+_rec("ShCAST", "CAST_VK", "insertion", "WT", 5000, "ecoli_genome", "E_coli", "bacterial", "plasmid", 80, "up to 80%",
+     "10.1126/science.aax9181",
+     "ShCAST integrates DNA into unique sites in the E. coli genome with frequencies of up to 80% without positive selection")
+_rec("PseCAST", "CAST_VK", "insertion", "WT", 1000, "ALB", "HEK293T", "human", "plasmid", 0.023, "0.023%",
+     "10.1126/science.adt5199", "5.7% targeted integration efficiency...compared to 0.023% for wild-type PseCAST (ALB)")
+_rec("evoCAST", "CAST_VK", "insertion", "evolved", 1000, "ALB", "HEK293T", "human", "plasmid", 5.7, "5.7%",
+     "10.1126/science.adt5199", "5.7% targeted integration efficiency...compared to 0.023% for wild-type PseCAST (ALB)")
+_rec("evoCAST", "CAST_VK", "insertion", "evolved", 1000, "TRAC", "HEK293T", "human", "plasmid", 13, "13%",
+     "10.1126/science.adt5199", "13% integration...compared to 0.061% by wild-type PseCAST (TRAC)")
+_rec("evoCAST", "CAST_VK", "insertion", "evolved", 5000, "14loci_avg", "HEK293T", "human", "plasmid", 19, "~10-25%",
+     "10.1126/science.adt5199", "evoCAST averaged ~10-25% integration efficiencies of kilobase-size DNA cargoes across 14 tested human genomic sites; averaged 19%")
+
+# ---- Bridge recombinases, enIS621 (Nat Commun 2026, 10.1038/s41467-026-74164-z), ISCro4 (Science) ----
+_B = "10.1038/s41467-026-74164-z"
+_rec("enIS621-tebRNA", "bridge_IS110", "insertion", "engineered", 1000, "endogenous_avg", "HEK293T", "human",
+     "plasmid", 6.82, "6.82%", _B, "6.82% for enIS621-tebRNA (average, two endogenous loci, HEK293T)",
+     source_access="secondary")
+_rec("enIS621-tebRNA", "bridge_IS110", "insertion", "engineered", 1000, "endogenous_avg", "Jurkat", "human",
+     "plasmid", 14.76, "14.76%", _B, "14.76% for enIS621-tebRNA (Jurkat T cells)", source_access="secondary")
+_rec("enIS621-tebRNA", "bridge_IS110", "insertion", "engineered", 1000, "best_locus", "multi_human", "human",
+     "plasmid", 27.75, "up to 27.75%", _B, "integration rates up to 27.75% for kilobase-scale DNA cargos",
+     source_access="abstract")
+_rec("ISCro4", "bridge_IS110", "insertion", "WT", 1000, "human_genome", "human_cells", "human", "plasmid", 6,
+     ">6%", "10.1126/science.adz1884",
+     "facilitates donor DNA insertion at genomic sites with efficiencies surpassing 6% (Pelea et al. human cells)",
+     source_access="secondary")
+_rec("ISCro4", "bridge_IS110", "insertion", "engineered_DMS", 1000, "human_genome", "human_cells", "human",
+     "plasmid", 20, "up to 20%", "10.1126/science.adz0276",
+     "up to 20% insertion efficiency into the human genome and genome-wide specificity as high as 82% (Perry et al.)",
+     source_access="secondary", spec=82)
+_rec("IS621", "bridge_IS110", "insertion", "WT", 1000, "ecoli_genome", "E_coli", "bacterial", "plasmid", 60,
+     ">60%", "10.1038/s41586-024-07552-4",
+     "over 60% insertion efficiency of a desired gene in E. coli with over 94% specificity (Durrant et al.)",
+     source_access="secondary", spec=94)
+
+
+def records() -> pd.DataFrame:
+    """The curated writer-efficiency table (one row per measured condition), with full per-row provenance."""
+    return pd.DataFrame(_R, columns=COLUMNS)
+
+
+def human_cell(df: pd.DataFrame | None = None, strict: bool = False) -> pd.DataFrame:
+    """Human-cell rows only (excludes E. coli). `strict=True` keeps only pmc_verbatim + abstract sources
+    (drops secondary-source rows), the high-confidence subset for the strict benchmark."""
+    df = records() if df is None else df
+    out = df[df["organism"] == "human"].copy()
+    if strict:
+        out = out[out["source_access"].isin(["pmc_verbatim", "abstract"])]
+    return out.reset_index(drop=True)
+
+
+def build_parquet(out: str | Path | None = None) -> Path:
+    """Write the SHA-lockable dataset parquet (data/writer_efficiency.parquet)."""
+    out = Path(out) if out else project_root() / "data/writer_efficiency.parquet"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    records().to_parquet(out, index=False)
+    return out
+
+
+def provenance_summary() -> dict:
+    df = records()
+    return {"n_records": int(len(df)), "n_human": int((df["organism"] == "human").sum()),
+            "by_family": df["family"].value_counts().to_dict(),
+            "by_source_access": df["source_access"].value_counts().to_dict(),
+            "n_dois": int(df["doi"].nunique()), "loci": sorted(df["locus"].unique()),
+            "cell_types": sorted(df["cell_type"].unique())}

pen_stack/atlas/writer_predict.py

@@ -0,0 +1,229 @@
+"""Learned cross-family writer-efficiency predictor.
+
+Given `(family, variant-is-evolved, write-type, cargo size, locus context, cell-type class)`, predict the
+integration **efficiency (%)** with a conformal interval, candidate-flagged. The baseline-to-beat is the
+curated-KB prior (a writer's family-mean efficiency; the KB ranks by family). Pre-registered acceptance gate: the
+learned model ships ONLY if it beats the family-mean baseline on **held-out family AND held-out
+locus** (paired-bootstrap CI excludes 0); otherwise the KB ranking is retained and the negative is reported.
+Never tune to the test.
+
+This is a SMALL curated dataset (~38 human-cell records across 4 families), the benchmark + the dataset are the
+standalone contribution, and a negative result is an expected outcome. Features are deliberately
+interpretable (no ESM/Evo2 embeddings here, at N=38 they would overfit; variant *scoring* uses ESM3/Evo2 in
+`design.writer_variants`). Every efficiency is a real published number (see `writer_efficiency`); nothing invented.
+"""
+from __future__ import annotations
+
+import pickle
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from scipy.stats import spearmanr
+
+from pen_stack.atlas.writer_efficiency import human_cell
+from pen_stack.wgenome.uncertainty import ConformalRegressor
+
+# interpretable feature construction -------------------------------------------------------------
+_PRIMARY = {"primary_T", "primary_hepatocyte", "primary_fibroblast", "hepatocyte_in_vivo", "iPSC"}
+_AGG_LOCI = {"8loci_avg", "14loci_avg", "endogenous_avg", "best_locus", "human_genome", "multi_human"}
+
+
+def _evolved(variant: str) -> int:
+    v = str(variant).lower()
+    wt = v in ("wt", "wt_bxb1") or v.startswith("wt")
+    return 0 if wt else 1
+
+
+def _celltype_class(ct: str) -> str:
+    if ct in _PRIMARY:
+        return "primary_or_ipsc"
+    return "cell_line"
+
+
+def featurize(df: pd.DataFrame) -> pd.DataFrame:
+    f = pd.DataFrame(index=df.index)
+    f["family"] = df["family"].astype("category")
+    f["evolved"] = df["variant"].map(_evolved).astype(int)
+    f["log_cargo"] = np.log10(df["cargo_bp"].astype(float).clip(lower=1))
+    f["cell_class"] = df["cell_type"].map(_celltype_class).astype("category")
+    f["delivery"] = df["delivery"].astype("category")
+    return f
+
+
+def specific_locus_mask(df: pd.DataFrame) -> np.ndarray:
+    """True for rows at a SPECIFIC named locus (excludes aggregate/avg/genome-wide pseudo-loci)."""
+    return ~df["locus"].isin(_AGG_LOCI)
+
+
+# baselines --------------------------------------------------------------------------------------
+def _family_mean_baseline(train: pd.DataFrame, test: pd.DataFrame) -> np.ndarray:
+    """The KB prior: predict a row's efficiency as the train family-mean (falls back to the global train mean
+    for an unseen family, which is exactly what happens under held-out-family)."""
+    gm = float(train["efficiency_pct"].mean())
+    fam_mean = train.groupby("family")["efficiency_pct"].mean().to_dict()
+    return np.array([fam_mean.get(f, gm) for f in test["family"]], float)
+
+
+# model ------------------------------------------------------------------------------------------
+def _fit_gbr(Xtr, ytr, seed: int = 42):
+    """Small, regularized GBR, appropriate for ~38 points (shallow, few estimators). One-hot the categoricals."""
+    from sklearn.ensemble import HistGradientBoostingRegressor
+    m = HistGradientBoostingRegressor(max_depth=2, max_iter=120, learning_rate=0.05,
+                                      l2_regularization=1.0, min_samples_leaf=4, random_state=seed)
+    m.fit(Xtr, ytr)
+    return m
+
+
+def _encode(f_train: pd.DataFrame, f_test: pd.DataFrame):
+    """One-hot encode categoricals consistently across train/test."""
+    cat = ["family", "cell_class", "delivery"]
+    num = ["evolved", "log_cargo"]
+    Xtr = pd.get_dummies(f_train[cat + num], columns=cat).astype(float)
+    Xte = pd.get_dummies(f_test[cat + num], columns=cat).reindex(columns=Xtr.columns, fill_value=0.0).astype(float)
+    return Xtr, Xte
+
+
+@dataclass
+class WriterEfficiencyModel:
+    columns: list[str] = field(default_factory=list)
+    model: object | None = None
+    conformal: ConformalRegressor | None = None
+    meta: dict = field(default_factory=dict)
+
+    def fit(self, df: pd.DataFrame | None = None, seed: int = 42) -> "WriterEfficiencyModel":
+        df = human_cell(strict=False) if df is None else df
+        f = featurize(df)
+        X = pd.get_dummies(f[["family", "cell_class", "delivery", "evolved", "log_cargo"]],
+                           columns=["family", "cell_class", "delivery"]).astype(float)
+        self.columns = list(X.columns)
+        self.model = _fit_gbr(X, df["efficiency_pct"].to_numpy(), seed)
+        self.meta = {"n": int(len(df)), "families": sorted(df["family"].unique()),
+                     "efficiency_units": "percent_integration"}
+        return self
+
+    def _design_matrix(self, df: pd.DataFrame) -> pd.DataFrame:
+        f = featurize(df)
+        X = pd.get_dummies(f[["family", "cell_class", "delivery", "evolved", "log_cargo"]],
+                           columns=["family", "cell_class", "delivery"]).astype(float)
+        return X.reindex(columns=self.columns, fill_value=0.0)
+
+    def predict(self, df: pd.DataFrame) -> np.ndarray:
+        return self.model.predict(self._design_matrix(df))
+
+    def predict_interval(self, df: pd.DataFrame) -> dict:
+        # Integration efficiency is a percentage, so the SERVED point estimate is clamped to the physical [0, 100]
+        # range (the regressor can extrapolate outside it on an out-of-distribution request), matching the interval
+        # bounds. The interval stays centred on the raw prediction, and `predict` is left unclamped so the
+        # pre-registered held-out evaluation scores the model exactly as trained.
+        yhat = self.predict(df)
+        if self.conformal is None or not np.isfinite(self.conformal.qhat):
+            return {"yhat": np.clip(yhat, 0, 100), "lo": None, "hi": None, "interval_kind": "uncalibrated"}
+        lo, hi = self.conformal.interval(yhat)
+        return {"yhat": np.clip(yhat, 0, 100), "lo": np.clip(lo, 0, 100), "hi": np.clip(hi, 0, 100),
+                "nominal_coverage": 1 - self.conformal.alpha,
+                "interval_kind": "trained split-conformal (% integration; candidate)"}
+
+    def save(self, path: str | Path) -> str:
+        path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "wb") as fh:
+            pickle.dump({"columns": self.columns, "model": self.model, "conformal": self.conformal,
+                         "meta": self.meta}, fh)
+        return str(path)
+
+    @classmethod
+    def load(cls, path: str | Path) -> "WriterEfficiencyModel":
+        with open(path, "rb") as fh:
+            return cls(**pickle.load(fh))
+
+
+# evaluation: leave-one-family-out + leave-one-locus-out, learned vs KB family-mean baseline -----
+def _loo_groups(df: pd.DataFrame, by: str):
+    vals = df[by].to_numpy()
+    for g in pd.unique(vals):
+        te = np.where(vals == g)[0]
+        tr = np.where(vals != g)[0]
+        if len(tr) >= 6 and len(te) >= 1:
+            yield g, tr, te
+
+
+def _boot_delta(model_err, base_err, reps=2000, seed=20260620):
+    """Paired bootstrap 95% CI on mean(base_err) - mean(model_err) (positive = model better, lower error)."""
+    rng = np.random.default_rng(seed)
+    me, be = np.asarray(model_err, float), np.asarray(base_err, float)
+    n = len(me)
+    d = []
+    for _ in range(reps):
+        idx = rng.integers(0, n, n)
+        d.append(float(np.mean(be[idx]) - np.mean(me[idx])))
+    lo, hi = np.percentile(d, [2.5, 97.5])
+    return {"delta_mean_abs_err_reduction": float(np.mean(d)), "ci95": [float(lo), float(hi)],
+            "model_beats_baseline": bool(lo > 0)}
+
+
+def evaluate(df: pd.DataFrame | None = None, seed: int = 42) -> dict:
+    """Held-out-family AND held-out-locus CV: learned model vs the KB family-mean baseline. Reports Spearman,
+    MAE, and the paired-bootstrap CI on the MAE reduction. The acceptance gate = beats baseline on BOTH axes (CI excludes 0)."""
+    df = human_cell(strict=False).reset_index(drop=True) if df is None else df.reset_index(drop=True)
+    out = {"n": int(len(df)), "n_families": int(df["family"].nunique())}
+    for axis, by, sub in (("held_out_family", "family", df),
+                          ("held_out_locus", "locus", df[specific_locus_mask(df)].reset_index(drop=True))):
+        y_true, y_model, y_base = [], [], []
+        for _g, tr, te in _loo_groups(sub, by):
+            tr_df, te_df = sub.iloc[tr], sub.iloc[te]
+            f = featurize(tr_df)
+            Xtr = pd.get_dummies(f[["family", "cell_class", "delivery", "evolved", "log_cargo"]],
+                                 columns=["family", "cell_class", "delivery"]).astype(float)
+            m = _fit_gbr(Xtr, tr_df["efficiency_pct"].to_numpy(), seed)
+            fte = featurize(te_df)
+            Xte = pd.get_dummies(fte[["family", "cell_class", "delivery", "evolved", "log_cargo"]],
+                                 columns=["family", "cell_class", "delivery"]).reindex(columns=Xtr.columns,
+                                                                                       fill_value=0.0).astype(float)
+            y_true.extend(te_df["efficiency_pct"].tolist())
+            y_model.extend(m.predict(Xte).tolist())
+            y_base.extend(_family_mean_baseline(tr_df, te_df).tolist())
+        y_true, y_model, y_base = map(np.array, (y_true, y_model, y_base))
+        if len(y_true) < 3:
+            out[axis] = {"available": False, "n": int(len(y_true))}
+            continue
+        me = np.abs(y_true - y_model)
+        be = np.abs(y_true - y_base)
+        out[axis] = {
+            "n": int(len(y_true)),
+            "mae_model": float(np.mean(me)), "mae_baseline_family_mean": float(np.mean(be)),
+            "spearman_model": float(spearmanr(y_model, y_true).statistic),
+            "spearman_baseline": float(spearmanr(y_base, y_true).statistic),
+            "delta": _boot_delta(me, be, seed=seed),
+        }
+    mf = out.get("held_out_family", {}).get("delta", {}).get("model_beats_baseline", False)
+    ml = out.get("held_out_locus", {}).get("delta", {}).get("model_beats_baseline", False)
+    out["gate_C_G2"] = {
+        "beats_on_held_out_family": bool(mf), "beats_on_held_out_locus": bool(ml),
+        "ship_learned_model": bool(mf and ml),
+        "verdict": ("learned predictor beats the KB family-mean baseline on BOTH held-out family AND locus, "
+                    "serve it (candidate-flagged)" if (mf and ml) else
+                    "learned predictor does NOT beat the KB baseline on both axes at this N, RETAIN the KB "
+                    "ranking, report the negative; the curated dataset + Writer-Efficiency Bench are the "
+                    "contribution (the pre-registered outcome)")}
+    return out
+
+
+def calibrate(df: pd.DataFrame | None = None, alpha: float = 0.10, seed: int = 42) -> ConformalRegressor:
+    """Leave-one-family-out OOF residuals → split-conformal interval on the % scale (family-blocked)."""
+    df = human_cell(strict=False).reset_index(drop=True) if df is None else df.reset_index(drop=True)
+    y_true, y_hat = [], []
+    for _g, tr, te in _loo_groups(df, "family"):
+        tr_df, te_df = df.iloc[tr], df.iloc[te]
+        f = featurize(tr_df)
+        Xtr = pd.get_dummies(f[["family", "cell_class", "delivery", "evolved", "log_cargo"]],
+                             columns=["family", "cell_class", "delivery"]).astype(float)
+        m = _fit_gbr(Xtr, tr_df["efficiency_pct"].to_numpy(), seed)
+        fte = featurize(te_df)
+        Xte = pd.get_dummies(fte[["family", "cell_class", "delivery", "evolved", "log_cargo"]],
+                             columns=["family", "cell_class", "delivery"]).reindex(columns=Xtr.columns,
+                                                                                   fill_value=0.0).astype(float)
+        y_true.extend(te_df["efficiency_pct"].tolist())
+        y_hat.extend(m.predict(Xte).tolist())
+    return ConformalRegressor(alpha=alpha).calibrate(np.array(y_true), np.array(y_hat))

pen_stack/atlas/writer_recommend.py

@@ -0,0 +1,170 @@
+"""Writer recommender, the surface that ties writer selection together.
+
+Given a write request `(write-type, cargo size, cell type, optional target/donor sequences)`, returns ranked
+writer families with:
+  * the **KB readiness** (PRIMARY ranking, the curated Writer Atlas; retained per the pre-registered gate),
+  * a **predicted efficiency + conformal interval**, candidate-flagged (the learned predictor, an advisory, NOT
+    the authoritative ranking, because at N=42/4-families it does not beat the KB baseline on held-out family),
+  * **cargo-capacity fit** (does the family carry this cargo?),
+  * an **auto-designed guide** (bridge RNA / pegRNA+attB) when sequences are supplied.
+
+Scope: efficiency predictions are candidates with intervals; the KB ranking is the grounded primary. No
+fabricated efficiency is ever emitted (the predictor is trained only on the curated real dataset).
+"""
+from __future__ import annotations
+
+from functools import lru_cache
+
+import pandas as pd
+
+from pen_stack._resources import project_root
+
+
+@lru_cache(maxsize=1)
+def _wtkb() -> pd.DataFrame:
+    return pd.read_parquet(project_root() / "pen_stack/atlas/wtkb.parquet")
+
+
+@lru_cache(maxsize=1)
+def _eff_model():
+    """Load the learned efficiency model artifact if present; else None (KB-only recommendation)."""
+    from pen_stack.atlas.writer_predict import WriterEfficiencyModel
+    p = project_root() / "models/writer_eff.pkl"
+    try:
+        return WriterEfficiencyModel.load(p) if p.exists() else None
+    except Exception: # noqa: BLE001
+        return None
+
+
+def _kb_readiness(row) -> float:
+    """A transparent KB readiness in [0,1] from the curated atlas fields: DSB-free (writing-relevant) + measured
+    human-cell activity + cargo headroom. PRIMARY ranking signal (grounded, not learned)."""
+    dsb_free = 1.0 if bool(row.get("dsb_free")) else 0.0
+    measured = 1.0 if str(row.get("confidence")) == "measured" else (0.5 if str(row.get("confidence")) == "inferred" else 0.25)
+    act = str(row.get("human_cell_activity") or "").lower()
+    human = 1.0 if ("%" in act or "high" in act) and "not measured" not in act and "low" not in act else (
+        0.3 if "low" in act or "not measured" in act else 0.6)
+    return round(0.4 * dsb_free + 0.35 * measured + 0.25 * human, 3)
+
+
+# Mechanism-grounded write-type suitability. A cargo write (insertion / landing-pad) NEEDS a cargo-carrying writer;
+# an excision NEEDS a programmable cutter. This is molecular MECHANISM (a cargo installer vs a cutter), NOT an
+# efficacy prediction -- a transparent, labelled tier so the write-type selector actually shapes the ranking.
+_CARGO_WRITES = frozenset({"insertion", "knock_in_with_disruption", "landing_pad_insertion", "high_durability_insertion"})
+_EXCISION_WRITES = frozenset({"excision", "repeat_excision", "regulatory_element_excision", "regulatory_excision"})
+WRITE_TYPES = _CARGO_WRITES | _EXCISION_WRITES | frozenset({"inversion"})
+
+
+# Families that catalyse site-specific INVERSION (reversible recombination between two inverted sites): the serine
+# integrases (invert between inverted att) and the IS110/IS1111 bridge recombinases (Durrant 2024); a PE-installed
+# att inherits the serine mechanism. The CAST transposase is unidirectional INSERTION only, and DSB nucleases invert
+# only imprecisely via dual cuts -- so those are not the mechanism for a clean inversion.
+_INVERSION_RECOMBINASES = frozenset({"serine_integrase", "PE_integrase", "bridge_IS110", "seek_IS1111"})
+
+
+def _write_type_suitability(write_type: str, family: str, carries_cargo: bool) -> int:
+    """Mechanism suitability TIER (1 = the right mechanism class for this write type, 0 = a poorer fit). Grounded
+    in mechanism, never an efficacy claim: a cargo write needs a cargo carrier; an excision needs a programmable
+    cutter; an inversion needs a site-specific recombinase."""
+    if write_type in _CARGO_WRITES:
+        return 1 if carries_cargo else 0             # nucleases can knock-in via HDR but do not deliver large cargo
+    if write_type in _EXCISION_WRITES:
+        return 0 if carries_cargo else 1             # a cut/excise favours a programmable cutter, not an installer
+    if write_type == "inversion":
+        return 1 if family in _INVERSION_RECOMBINASES else 0  # only site-specific recombinases invert cleanly
+    return 1                                         # other: neutral (no grounded reweight)
+
+
+def recommend_writers(request: dict, top_k: int = 5) -> dict:
+    """Rank writer families for a write request. KB readiness is primary; learned efficiency is a candidate
+    advisory; guide design attached when sequences are supplied."""
+    cargo_bp = float(request.get("cargo_bp") or 0)
+    cell_type = request.get("cell_type") or "HEK293T"
+    # `write_type` is the documented key; `intent` is accepted as a synonym because the planner CLI names the same
+    # idea `--intent`, so a request that uses it is honoured instead of silently falling back to the default
+    # insertion. An unrecognized value still ranks neutrally, but says so via `write_type_recognized` below rather
+    # than looking like a shaped ranking.
+    write_type = request.get("write_type") or request.get("intent") or "insertion"
+    target_seq, donor_seq = request.get("target_seq"), request.get("donor_seq")
+    model = _eff_model()
+    rows = []
+    for _, r in _wtkb().iterrows():
+        cap = float(r["cargo_capacity_bp"]) if pd.notna(r.get("cargo_capacity_bp")) else 0.0
+        cargo_fit = (cap >= cargo_bp) if (cargo_bp and cap) else None
+        rec = {
+            "family": r["family"], "representative_system": r["representative_system"],
+            "kb_readiness": _kb_readiness(r), "dsb_free": bool(r.get("dsb_free")),
+            "write_type_suitability": _write_type_suitability(write_type, r["family"], cap > 0),  # mechanism fit
+            "cargo_capacity_bp": int(cap) if cap else None,
+            "cargo_fit": cargo_fit, "confidence": str(r.get("confidence")),
+            "key_dois": [str(x) for x in r.get("key_dois")] if r.get("key_dois") is not None else [],
+        }
+        # learned efficiency (candidate advisory), ONLY for families present in the curated dataset (never
+        # extrapolate an efficiency to an unseen family; KB-only for those, no fabrication).
+        if model is not None and r["family"] in model.meta.get("families", []):
+            try:
+                df = pd.DataFrame([{"family": r["family"], "variant": "engineered",
+                                    "cargo_bp": max(cargo_bp, 1000), "cell_type": cell_type,
+                                    "delivery": "plasmid"}])
+                iv = model.predict_interval(df)
+                if iv.get("lo") is not None:
+                    rec["predicted_efficiency_pct"] = round(float(iv["yhat"][0]), 1)
+                    rec["efficiency_interval_pct"] = [round(float(iv["lo"][0]), 1), round(float(iv["hi"][0]), 1)]
+                    rec["efficiency_kind"] = "candidate (learned; interval = trained split-conformal)"
+            except Exception: # noqa: BLE001 - family unseen by the model -> KB-only, never fabricate
+                pass
+        # guide design when sequences supplied
+        if target_seq:
+            from pen_stack.atlas.guide_design import design_guide_for_writer
+            g = design_guide_for_writer(r["family"], target_seq, donor_seq)
+            if g.get("available"):
+                # surface the DESIGNED sequences (spacer / attB / bridge loops), not just a bare flag
+                rec["guide_design"] = {"type": g["design_type"], "feasible": g.get("feasible"),
+                                       "design": g.get("design")}
+        rows.append(rec)
+
+    # rank by mechanism suitability for the write type FIRST (grounded tier), then the grounded KB readiness within
+    # tier, then cargo fit. For the default insertion this is unchanged (cargo carriers already lead); an excision
+    # promotes programmable cutters over cargo installers.
+    rows.sort(key=lambda x: (x["write_type_suitability"], x["kb_readiness"], x.get("cargo_fit") is True), reverse=True)
+    if cargo_bp:
+        rows = [x for x in rows if x["cargo_fit"] is not False] + [x for x in rows if x["cargo_fit"] is False]
+    wt_note = None
+    if write_type in _EXCISION_WRITES:
+        wt_note = (f"Ranking shaped for a {write_type.replace('_', ' ')}: programmable cutters are favoured over "
+                   "cargo-installing integrases, which are the tool for insertion, not excision. Mechanism-based "
+                   "modifier, not an efficacy claim.")
+    elif write_type in _CARGO_WRITES:
+        wt_note = (f"Ranking shaped for a {write_type.replace('_', ' ')}: cargo-carrying writers (integrases / "
+                   "transposons) are favoured; nucleases can knock in via HDR but do not deliver a large cargo.")
+    elif write_type == "inversion":
+        wt_note = ("Ranking shaped for an inversion: site-specific recombinases (serine integrases, IS110/IS1111 "
+                   "bridge recombinases) that invert between inverted sites are favoured; the CAST transposase is "
+                   "insertion-only and DSB nucleases invert only imprecisely via dual cuts. Mechanism-based "
+                   "modifier, not an efficacy claim.")
+    # Cargo-capacity check: when a cargo is requested that NO cargo-carrying writer in the atlas can hold, the
+    # top of the ranking is nucleases (which don't deliver cargo, cargo_fit=n/a) or over-capacity integrases --
+    # surface that explicitly so Cas9 does not read as a silent #1 recommendation for an impossible payload.
+    cargo_warning = None
+    if cargo_bp:
+        caps = [x["cargo_capacity_bp"] for x in rows if x["cargo_capacity_bp"]]
+        max_cap = max(caps) if caps else 0
+        if max_cap and cargo_bp > max_cap:
+            cargo_warning = (
+                f"No writer in the atlas carries a {int(cargo_bp):,} bp cargo -- the largest cargo capacity is "
+                f"{int(max_cap):,} bp. Families marked 'over capacity' exceed their limit; nucleases (cargo fit "
+                f"n/a) do not deliver cargo at all. Consider splitting the payload or a dual-vector strategy.")
+    return {
+        "request": {"write_type": write_type, "write_type_recognized": write_type in WRITE_TYPES,
+                    "cargo_bp": cargo_bp or None, "cell_type": cell_type},
+        "ranking_basis": "mechanism suitability for the write type, then KB readiness (grounded); predicted "
+                         "efficiency is a candidate advisory",
+        "recommendations": rows[:top_k],
+        "n_families": len(rows),
+        "write_type_note": wt_note,
+        "cargo_capacity_warning": cargo_warning,
+        "no_fabrication": True,
+        "note": "Writer Atlas KB ranking is the grounded primary; learned efficiencies are candidates with "
+                "conformal intervals (the learned model does not beat the KB baseline on held-out family "
+                "at this N). Guide designs are candidates requiring empirical validation.",
+    }