pen-stack 3.1.0__py3-none-any.whl

pen_stack/atlas/expand.py

@@ -0,0 +1,190 @@
+"""Expand the Writer Atlas across families (Phase 2, Step 2.1).
+
+Grow the Phase-0 curated 8-family core into a comprehensive cross-family catalogue: ingest ortholog
+sets at scale (the IS110/IS1111 superfamily, CAST, large serine integrases, Cas12a, TnpB/Fanzor) from
+UniProt, place every entry on the WT-KB targeting axes by *inheriting* family-level metadata from the
+Phase-0 ``wtkb.parquet`` (single source of truth - the classifier/scorer must not re-derive it), and
+tag each row with an explicit ``confidence`` (measured / inferred / predicted) and provenance.
+
+Heavy per-ortholog featurisation (ESM embeddings for mechanism classification at scale, Step 2.2) runs
+in Docker on the GPU; this module only assembles the *catalogue* metadata (lightweight, network-bound).
+
+Inputs : configs/atlas_families.yaml, pen_stack/atlas/wtkb.parquet, UniProt REST.
+Outputs: pen_stack/atlas/atlas.parquet (one row per system), cached TSVs under data/external/atlas/.
+"""
+from __future__ import annotations
+
+import time
+import urllib.parse
+import urllib.request
+from io import StringIO
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+_ROOT = Path(__file__).resolve().parents[2]
+_CFG = _ROOT / "configs" / "atlas_families.yaml"
+_WTKB = _ROOT / "pen_stack" / "atlas" / "wtkb.parquet"
+_CACHE = _ROOT / "data" / "external" / "atlas"
+_OUT = _ROOT / "pen_stack" / "atlas" / "atlas.parquet"
+
+_UNIPROT_STREAM = "https://rest.uniprot.org/uniprotkb/stream"
+
+# WT-KB family-level fields every atlas row inherits (so targeting metadata has ONE source).
+_INHERIT = [
+    "mechanism_bucket", "targeting_modality", "target_site_spec", "guide_architecture",
+    "cargo_mechanism", "cargo_capacity_bp", "dsb_free", "reachability_tier",
+    "reachability_constraints",
+]
+
+
+def load_config(path: str | Path = _CFG) -> dict:
+    return yaml.safe_load(Path(path).read_text(encoding="utf-8"))
+
+
+def fetch_uniprot(query: str, fields: str, cache: Path, timeout: int = 120,
+                  retries: int = 3) -> pd.DataFrame:
+    """Stream a UniProt query to TSV (cached). Returns the raw per-accession metadata frame."""
+    cache.parent.mkdir(parents=True, exist_ok=True)
+    if cache.exists():
+        return pd.read_csv(cache, sep="\t", dtype=str)
+    params = {"query": query, "format": "tsv", "fields": fields}
+    url = _UNIPROT_STREAM + "?" + urllib.parse.urlencode(params)
+    last = None
+    for attempt in range(retries):
+        try:
+            with urllib.request.urlopen(url, timeout=timeout) as r:
+                text = r.read().decode("utf-8")
+            df = pd.read_csv(StringIO(text), sep="\t", dtype=str)
+            cache.write_text(text, encoding="utf-8")
+            return df
+        except Exception as e:  # noqa: BLE001 - network is best-effort; surfaced after retries
+            last = e
+            time.sleep(2 * (attempt + 1))
+    raise RuntimeError(f"UniProt fetch failed for query {query!r}: {last}")
+
+
+def _orthologs_for_family(fam_key: str, fam: dict, fields: str, wtkb_row: pd.Series,
+                          cache_dir: Path) -> pd.DataFrame:
+    """One ortholog table for a family, inheriting WT-KB targeting metadata by family."""
+    cache = cache_dir / f"{fam_key}.tsv"
+    raw = fetch_uniprot(fam["query"], fields, cache)
+    cap = int(fam.get("cap", len(raw)))
+    raw = raw.head(cap).copy()
+
+    # UniProt TSV column names (from the requested fields)
+    acc_col = "Entry"
+    org_col = "Organism"
+    len_col = "Length"
+    df = pd.DataFrame({
+        "representative_system": raw.get(acc_col),
+        "uniprot": raw.get(acc_col),
+        "organism": raw.get(org_col),
+        "length_aa": pd.to_numeric(raw.get(len_col), errors="coerce"),
+    })
+    df["family"] = fam["wtkb_family"]
+    df["pfam_signature"] = [list(fam["pfam_signature"])] * len(df)
+    df["confidence"] = fam.get("default_confidence", "predicted")
+    df["human_cell_activity"] = "not measured (sequence homolog)"
+    df["key_dois"] = [[fam["discovery_doi"]]] * len(df)
+    df["entry_kind"] = "ortholog"
+    # inherit targeting metadata from the WT-KB family row (single source of truth)
+    for col in _INHERIT:
+        df[col] = wtkb_row.get(col)
+    return df
+
+
+def _curated_rows(cfg: dict, wtkb: pd.DataFrame) -> pd.DataFrame:
+    """Named, characterised systems: the 8 WT-KB families themselves + extra reps from config."""
+    rows = []
+    # (a) the WT-KB curated core - measured/inferred, full targeting spec
+    for _, w in wtkb.iterrows():
+        rows.append({
+            "representative_system": w["representative_system"],
+            "uniprot": w.get("uniprot"),
+            "organism": None,
+            "length_aa": w.get("length_aa"),
+            "family": w["family"],
+            "pfam_signature": list(w["pfam_signature"]) if w.get("pfam_signature") is not None else [],
+            "confidence": w.get("confidence", "measured"),
+            "human_cell_activity": w.get("human_cell_activity"),
+            "key_dois": list(w["key_dois"]) if w.get("key_dois") is not None else [],
+            "entry_kind": "curated_core",
+            **{c: w.get(c) for c in _INHERIT},
+        })
+    # (b) extra curated representatives (named systems w/o a clean single-Pfam query)
+    wt_by_fam = wtkb.set_index("family")
+    for rep in cfg.get("curated_representatives", []):
+        fam = rep["family"]
+        wrow = wt_by_fam.loc[fam] if fam in wt_by_fam.index else pd.Series(dtype=object)
+        if isinstance(wrow, pd.DataFrame):
+            wrow = wrow.iloc[0]
+        rows.append({
+            "representative_system": rep["representative_system"],
+            "uniprot": rep.get("uniprot"),
+            "organism": None,
+            "length_aa": rep.get("length_aa") or (wrow.get("length_aa") if len(wrow) else None),
+            "family": fam,
+            "pfam_signature": list(wrow.get("pfam_signature")) if len(wrow) and wrow.get("pfam_signature") is not None else [],
+            "confidence": rep.get("confidence", "inferred"),
+            "human_cell_activity": rep.get("human_cell_activity"),
+            "key_dois": list(rep.get("key_dois", [])),
+            "entry_kind": "curated_rep",
+            **{c: (wrow.get(c) if len(wrow) else None) for c in _INHERIT},
+        })
+    return pd.DataFrame(rows)
+
+
+def confidence_tag(row: pd.Series) -> str:
+    """measured (human-cell data) > inferred (characterised, non-human) > predicted (homolog only)."""
+    c = row.get("confidence")
+    if c in {"measured", "inferred", "predicted"}:
+        return c
+    hca = (row.get("human_cell_activity") or "").lower()
+    if "human cell" in hca and "not measured" not in hca:
+        return "measured"
+    return "predicted"
+
+
+def build_atlas(cfg_path: str | Path = _CFG, wtkb_path: str | Path = _WTKB,
+                out: str | Path = _OUT, cache_dir: str | Path = _CACHE,
+                offline_ok: bool = False) -> pd.DataFrame:
+    cfg = load_config(cfg_path)
+    fields = cfg["defaults"]["uniprot_fields"]
+    wtkb = pd.read_parquet(wtkb_path)
+    wt_by_fam = wtkb.drop_duplicates("family").set_index("family")
+    cache_dir = Path(cache_dir)
+
+    tables: list[pd.DataFrame] = [_curated_rows(cfg, wtkb)]
+    for fam_key, fam in cfg["families"].items():
+        wrow = wt_by_fam.loc[fam["wtkb_family"]] if fam["wtkb_family"] in wt_by_fam.index else pd.Series(dtype=object)
+        try:
+            tables.append(_orthologs_for_family(fam_key, fam, fields, wrow, cache_dir))
+        except Exception as e:  # noqa: BLE001
+            if offline_ok:
+                print(f"[expand] skip {fam_key} (offline_ok): {e}")
+                continue
+            raise
+
+    atlas = pd.concat(tables, ignore_index=True)
+    # named curated rows win over a homolog row for the same accession. Dedup ONLY among rows that
+    # carry a UniProt id - rows without one (seekRNA, PASTE, ShCAST, Bxb1, ISPpu10) are all distinct
+    # systems and must never collapse together (pandas treats every NaN as equal under drop_duplicates).
+    atlas["_pri"] = atlas["entry_kind"].map({"curated_core": 0, "curated_rep": 1, "ortholog": 2})
+    has_acc = atlas["uniprot"].notna() & (atlas["uniprot"].astype(str).str.strip() != "")
+    with_acc = (atlas[has_acc].sort_values("_pri")
+                .drop_duplicates(subset=["uniprot"], keep="first"))
+    atlas = pd.concat([with_acc, atlas[~has_acc]], ignore_index=True).drop(columns="_pri")
+    atlas["confidence"] = atlas.apply(confidence_tag, axis=1)
+    atlas = atlas.reset_index(drop=True)
+
+    Path(out).parent.mkdir(parents=True, exist_ok=True)
+    atlas.to_parquet(out, index=False)
+    return atlas
+
+
+if __name__ == "__main__":  # pragma: no cover
+    a = build_atlas()
+    print(f"atlas rows: {len(a):,}")
+    print(a.groupby(["family", "confidence"]).size())

pen_stack/atlas/schema.py

@@ -0,0 +1,59 @@
+"""WT-KB schema - the Writer-Targeting Knowledge Base row model (Phase 0, Step 0.2).
+
+One row per writer family/representative system: its targeting requirements and a reachability tier.
+This is the spine of the Writer Atlas and the reachability layer of the Writable Genome. Every
+targeting field must carry at least one DOI in ``key_dois`` - nothing is asserted without a citation.
+"""
+from __future__ import annotations
+
+from enum import Enum
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class MechBucket(str, Enum):
+    DSB_NUCLEASE = "DSB_NUCLEASE"
+    DSB_FREE_RECOMBINASE = "DSB_FREE_TRANSEST_RECOMBINASE"
+    TRANSPOSASE = "TRANSPOSASE"
+
+
+class Tier(str, Enum):
+    T1 = "Tier1_scannable"          # bridge/seek cores, PE-installable att
+    T2 = "Tier2_context_candidate"  # CAST, native pseudo-att integrases (candidate - requires validation)
+    T3 = "Tier3_not_predictable"    # retroelement preferences
+
+
+class Confidence(str, Enum):
+    MEASURED = "measured"
+    INFERRED = "inferred"
+    PREDICTED = "predicted"
+
+
+class WriterEntry(BaseModel):
+    model_config = ConfigDict(use_enum_values=True)
+
+    family: str
+    representative_system: str
+    uniprot: str | None = None
+    mechanism_bucket: MechBucket
+    pfam_signature: list[str]
+    targeting_modality: str             # RNA-guided | fixed-att | DDE-spacing | PE-installable
+    target_site_spec: str               # e.g. "bipartite ~14 nt, central CT dinucleotide core"
+    guide_architecture: str             # e.g. "bridge RNA: TBL(LTG/RTG)+DBL(LDG/RDG)"
+    cargo_mechanism: str                # intrinsic | fixed-donor | templated
+    cargo_capacity_bp: int | None = None
+    dsb_free: bool
+    length_aa: int | None = None
+    human_cell_activity: str | None = None   # measured value + source, or "not measured"
+    deliverability: str                 # AAV | split-AAV | mRNA-RNP
+    reachability_tier: Tier
+    reachability_constraints: str       # rules a genome scan must apply
+    confidence: Confidence = Confidence.MEASURED
+    key_dois: list[str] = Field(min_length=1)
+
+    @field_validator("key_dois")
+    @classmethod
+    def _nonempty_dois(cls, v: list[str]) -> list[str]:
+        if not v or not all(d.strip() for d in v):
+            raise ValueError("every WT-KB row must carry >=1 non-empty DOI (sourcing rule)")
+        return v

pen_stack/atlas/scorecard.py

@@ -0,0 +1,134 @@
+"""Descriptive writer scorecard (Phase 0, Step 0.5).
+
+Reframes the prior 5-gate "TRUE_WRITER certification" (circular - it pre-registered ISCro4 *by name*
+and depended on hand-set scores) into a transparent, DESCRIPTIVE scorecard computed from the
+re-grounded axes. No enzyme is named in any pre-registered prediction. We additionally report a
+*blind concordance* outcome: does the ranking place ISCro4 at the top of the bridge family using only
+generic measured axes (cell-based evidence, DSB-freeness, programmability, cargo) - without any
+ISCro4-specific value being asserted? This is reported, never asserted as an input.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import yaml
+
+_AXES_CFG = Path(__file__).resolve().parents[2] / "configs" / "score_axes.yaml"
+
+_EVIDENCE_COLS = ["has_biochemical", "has_structural", "has_computational", "has_cell_based"]
+
+# descriptive tier labels (NOT a certification; no enzyme is pre-named)
+T_DSB_DEPENDENT = "DSB_dependent"        # fails the necessary DSB-free gate (a "scissor")
+T_EMERGING = "emerging_writer"
+T_PROBABLE = "probable_writer"
+T_ESTABLISHED = "established_writer"      # DSB-free + fully programmable + native cargo + human-cell evidence
+
+
+def _thresholds(path: Path = _AXES_CFG) -> dict:
+    """Read v3.0 gate thresholds (defined on the RE-GROUNDED axis scales) from score_axes.yaml."""
+    if Path(path).exists():
+        g = yaml.safe_load(Path(path).read_text(encoding="utf-8")).get("gates_v3_0", {})
+        return {
+            "g1": g.get("g1_dsb_min", 0.95),
+            "g2": g.get("g2_prog_min", 0.95),
+            "g3": g.get("g3_cargo_min", 0.65),
+            "g4": g.get("g4_max_length_aa", 900),
+            "g5": g.get("g5_min_evidence", 2),
+        }
+    return {"g1": 0.95, "g2": 0.95, "g3": 0.65, "g4": 900, "g5": 2}
+
+
+def _evidence_count(row) -> int:
+    return int(sum(bool(row.get(c, False)) for c in _EVIDENCE_COLS))
+
+
+def _gates(row, th) -> dict:
+    g1 = float(row.get("s_dsb", 0) or 0) >= th["g1"]
+    g2 = float(row.get("S_Prog", 0) or 0) >= th["g2"]
+    g3 = (float(row.get("S_Cargo", 0) or 0) >= th["g3"]) and bool(row.get("intrinsic_cargo_mechanism", False))
+    length = row.get("length_aa")
+    g4 = (length is not None and not pd.isna(length) and float(length) <= th["g4"])
+    g5 = _evidence_count(row) >= th["g5"]
+    return {"g1": g1, "g2": g2, "g3": g3, "g4": g4, "g5": g5}
+
+
+def _descriptive_tier(row, th) -> str:
+    g = _gates(row, th)
+    if not g["g1"]:
+        return T_DSB_DEPENDENT                       # necessary gate (DSB-free) failed
+    qualifying = sum([g["g2"], g["g3"], g["g4"], g["g5"]])
+    has_cell = bool(row.get("has_cell_based", False))
+    if qualifying == 4 and has_cell:
+        return T_ESTABLISHED
+    if qualifying == 4 or (qualifying == 3 and has_cell):
+        return T_PROBABLE
+    if qualifying >= 1:
+        return T_EMERGING
+    return T_DSB_DEPENDENT
+
+
+def composite(row) -> float:
+    """Transparent composite; components stay visible on the scorecard.
+
+    Includes human-cell evidence (``has_cell_based``) as a generic readiness axis - this is the
+    signal that distinguishes the standout human-cell bridge recombinase. It is a generic column
+    present for every editor, NOT an ISCro4-specific asserted value (so the concordance stays blind).
+    """
+    parts = [
+        float(row.get("s_dsb", 0) or 0),
+        float(row.get("S_Prog", 0) or 0),
+        float(row.get("S_Cargo", 0) or 0),
+        _evidence_count(row) / 4.0,
+        float(bool(row.get("has_cell_based", False))),
+    ]
+    return float(np.mean(parts))
+
+
+def scorecard(universe_df: pd.DataFrame) -> pd.DataFrame:
+    th = _thresholds()
+    df = universe_df.copy()
+    df["evidence_count"] = df.apply(_evidence_count, axis=1)
+    df["S_composite"] = df.apply(composite, axis=1)
+    df["tier"] = df.apply(lambda r: _descriptive_tier(r, th), axis=1)
+    return df.sort_values("S_composite", ascending=False).reset_index(drop=True)
+
+
+def blind_concordance(scorecard_df: pd.DataFrame, family: str = "bridge_IS110",
+                      expected_top: str = "ISCro4") -> dict:
+    """Report (do NOT assert) whether the ranking places `expected_top` first within `family`,
+    using only generic measured axes. Returns the observed top + whether it matches.
+    Restricted to NATURAL editors (the concordance question is about natural systems)."""
+    sub = scorecard_df.query("family == @family")
+    if "source" in sub.columns:
+        sub = sub[sub["source"] == "natural"]
+    sub = sub.sort_values("S_composite", ascending=False)
+    if sub.empty:
+        return {"family": family, "top": None, "matches": False, "n": 0}
+    top = sub.iloc[0]["entity_id"]
+    return {"family": family, "top": top, "matches": (top == expected_top), "n": int(len(sub)),
+            "ranking": sub[["entity_id", "S_composite", "evidence_count"]].to_dict("records")}
+
+
+def ranking_stability(universe_df: pd.DataFrame, family: str = "bridge_IS110",
+                      expected_top: str = "ISCro4", n: int = 200, seed: int = 42) -> float:
+    """Fraction of randomly re-weighted composites under which `expected_top` stays family-top
+    (a lightweight sensitivity check, mirroring the prior ranking-stability analysis)."""
+    rng = np.random.default_rng(seed)
+    sub = universe_df.query("family == @family").copy()
+    if "source" in sub.columns:
+        sub = sub[sub["source"] == "natural"]
+    if sub.empty:
+        return 0.0
+    cols = ["s_dsb", "S_Prog", "S_Cargo"]
+    ev = sub.apply(_evidence_count, axis=1).to_numpy() / 4.0
+    cell = sub.get("has_cell_based", pd.Series([False] * len(sub))).fillna(False).astype(float).to_numpy()
+    X = np.column_stack([sub[c].fillna(0).astype(float).to_numpy() for c in cols] + [ev, cell])
+    wins = 0
+    for _ in range(n):
+        w = rng.dirichlet(np.ones(X.shape[1]))
+        scores = X @ w
+        if sub.iloc[int(scores.argmax())]["entity_id"] == expected_top:
+            wins += 1
+    return wins / n

pen_stack/atlas/universe.py

@@ -0,0 +1,75 @@
+"""Canonical universe assembly (Phase 0, Step 0.4).
+
+THE single entry point that joins the upstream editor universe + the WT-KB + the crosswalk and
+applies the re-grounded axes. The classifier, the scorer, and the scorecard must all consume the
+output of ``assemble()`` - never re-derive metadata independently (the prior PEN-DISCOVER vs
+PEN-COMPARE gate inconsistency must not recur). Cross-module consistency is asserted by
+``tests/unit/test_universe_consistency.py``.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+from pen_stack.score.recalibrate import load_axes_config, recalibrate_all
+
+_ROOT = Path(__file__).resolve().parents[2]
+_UNIVERSE = _ROOT / "data" / "curated" / "unified_editor_universe.parquet"
+_WTKB = _ROOT / "pen_stack" / "atlas" / "wtkb.parquet"
+_CROSSWALK = _ROOT / "configs" / "universe_crosswalk.yaml"
+
+
+def _load_crosswalk(path: Path = _CROSSWALK) -> pd.DataFrame:
+    cw = yaml.safe_load(path.read_text(encoding="utf-8"))["entity_to_family"]
+    return pd.DataFrame(
+        [{"entity_id": k, "family": v["family"], "targeting_modality": v["targeting_modality"]}
+         for k, v in cw.items()]
+    )
+
+
+def assemble(
+    universe_parquet: str | Path = _UNIVERSE,
+    wtkb_parquet: str | Path = _WTKB,
+    crosswalk_path: str | Path = _CROSSWALK,
+    out_parquet: str | Path | None = None,
+) -> pd.DataFrame:
+    uni = pd.read_parquet(universe_parquet)
+    wt = pd.read_parquet(wtkb_parquet)
+    cw = _load_crosswalk(Path(crosswalk_path))
+
+    # 1) attach family + modality to natural editors via the crosswalk
+    uni = uni.merge(cw, on="entity_id", how="left")
+
+    # 2) designs inherit their parent_editor's family + modality
+    if "parent_editor" in uni.columns:
+        parent_map = cw.set_index("entity_id")[["family", "targeting_modality"]]
+        need = uni["family"].isna() & uni["parent_editor"].notna()
+        for col in ("family", "targeting_modality"):
+            uni.loc[need, col] = uni.loc[need, "parent_editor"].map(parent_map[col])
+
+    # 3) bring WT-KB measured fields (cargo bp, reachability tier, dsb_free) in by family - single source
+    wt_fields = wt[["family", "cargo_capacity_bp", "reachability_tier", "dsb_free"]].drop_duplicates("family")
+    uni = uni.merge(wt_fields, on="family", how="left")
+
+    # 4) apply the re-grounded axes (length backfill + cargo + prog); NO per-enzyme overrides
+    uni = recalibrate_all(uni, load_axes_config())
+
+    if out_parquet:
+        Path(out_parquet).parent.mkdir(parents=True, exist_ok=True)
+        uni.to_parquet(out_parquet, index=False)
+    return uni
+
+
+# Axis/gate inputs that every downstream module must read from the canonical universe (not re-derive).
+CANONICAL_INPUTS = [
+    "entity_id", "source", "mechanism_class", "s_dsb", "S_Prog", "S_Cargo",
+    "length_aa", "intrinsic_cargo_mechanism", "cell_based_evidence",
+    "family", "targeting_modality", "reachability_tier",
+]
+
+
+def canonical_inputs(df: pd.DataFrame) -> pd.DataFrame:
+    """The exact metadata slice the classifier/scorer/scorecard must share."""
+    return df[[c for c in CANONICAL_INPUTS if c in df.columns]].copy()

pen_stack/atlas/variant_propose.py

@@ -0,0 +1,155 @@
+"""DMS-grounded variant proposal (Phase 2, Step 2.4) - replaces the failed de-novo chimera generation.
+
+Instead of speculative chimeras (PEN-ASSEMBLE produced 0 TRUE_WRITERs and was HPC-hungry/unvalidatable),
+propose *single/double point mutations* with a measured activity effect. **No chimeras are ever produced**
+- only point substitutions.
+
+The activity predictor is a pluggable ``VariantEffectModel``. The real model is ``DMSVariantEffectModel``,
+backed by the Perry 2025 deep mutational scan of ISCro4 (Table S3, delivered in Phase 1.5): it scores each
+substitution by its MEASURED activity Z-score. Fed that model, the framework's top proposals ARE the
+experimentally enhancing mutations - N322P (rank 1), H50K (rank 2), R278M - so it RECOVERS the known
+enhancers. Stated honestly per the program's framing: this is a useful catalogue feature that recovers
+KNOWN enhancers from the DMS; it is NOT a novel variant-design method and it is NOT a blind sequence-only
+prediction. For GENERATING new variants the established engine is EVOLVEpro - wrap it, do not rebuild.
+
+When the DMS is absent, a transparent physico-chemical *baseline* keeps the framework runnable (it makes
+no activity claim and must never be presented as the DMS model).
+
+Inputs : enzyme sequence; a VariantEffectModel (DMSVariantEffectModel, or the labelled baseline).
+Outputs: out/variant_proposals_<enzyme>.csv (ranked point mutations + measured effect).
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+import pandas as pd
+
+_AA = "ACDEFGHIKLMNPQRSTVWY"
+_OUT = Path(__file__).resolve().parents[2] / "out"
+
+# Kyte-Doolittle hydropathy + a coarse charge/volume signal, for the labelled baseline ONLY.
+_HYDRO = {"A": 1.8, "R": -4.5, "N": -3.5, "D": -3.5, "C": 2.5, "Q": -3.5, "E": -3.5, "G": -0.4,
+          "H": -3.2, "I": 4.5, "L": 3.8, "K": -3.9, "M": 1.9, "F": 2.8, "P": -1.6, "S": -0.8,
+          "T": -0.7, "W": -0.9, "Y": -1.3, "V": 4.2}
+
+
+@runtime_checkable
+class VariantEffectModel(Protocol):
+    """Predict a per-mutation activity gain. (i, wt, mut) -> predicted effect (higher = better)."""
+
+    name: str
+
+    def predict(self, seq: str, variants: list[tuple[int, str, str]]) -> list[float]: ...
+
+
+class BaselinePhysicoChemical:
+    """A transparent, NON-DMS placeholder predictor (Phase-1.5 supplies the real DMS model).
+
+    Scores a substitution by *conservativeness* (small hydropathy change ranks higher) - a deliberately
+    weak, documented heuristic so the proposal/validation framework is exercisable before Phase 1.5.
+    It makes no activity claim and must never be presented as the DMS predictor.
+    """
+
+    name = "baseline_physicochemical_placeholder"
+
+    def predict(self, seq: str, variants: list[tuple[int, str, str]]) -> list[float]:
+        return [-abs(_HYDRO.get(mut, 0.0) - _HYDRO.get(wt, 0.0)) for (_, wt, mut) in variants]
+
+
+class DMSVariantEffectModel:
+    """The REAL model: scores a substitution by its MEASURED activity Z-score from the Perry 2025 ISCro4
+    deep mutational scan (Table S3, Phase 1.5). Substitutions not present in the scan get a strongly
+    negative score (treated as unmeasured/non-enhancing). This recovers known enhancers; it is not a blind
+    sequence predictor (see module docstring). Requires the Perry tables locally (PEN_PERRY_DIR)."""
+
+    name = "perry2025_dms_iscro4"
+
+    def __init__(self) -> None:
+        from pen_stack.bridge.ingest import load_dms
+        dms = load_dms()
+        if dms.empty:
+            raise FileNotFoundError("Perry 2025 DMS (Table S3) not available; set PEN_PERRY_DIR")
+        z = pd.to_numeric(dms["Z_Score_wrt_WT"], errors="coerce")
+        self._z = dict(zip(dms["Mutation"].astype(str), z))
+
+    def predict(self, seq: str, variants: list[tuple[int, str, str]]) -> list[float]:
+        # variant key is wt + 1-based position + mut, e.g. "N322P"
+        return [self._z.get(f"{wt}{i + 1}{mut}", -9.9) for (i, wt, mut) in variants]
+
+
+def iscro4_sequence() -> str | None:
+    """ISCro4 recombinase sequence from Perry 2025 Table S1 (326 aa). None if absent."""
+    from pen_stack.bridge.ingest import load_screen
+    s1 = load_screen()
+    row = s1[s1["Name"].astype(str) == "ISCro4"] if not s1.empty else s1
+    return row.iloc[0]["Recombinase_Sequence"] if len(row) else None
+
+
+def propose_variants(seq: str, model: VariantEffectModel, top: int = 20,
+                     positions: list[int] | None = None) -> pd.DataFrame:
+    """Rank single point mutations by predicted activity gain. No chimeras - substitutions only."""
+    idxs = positions if positions is not None else range(len(seq))
+    cand = [(i, seq[i], aa) for i in idxs for aa in _AA if aa != seq[i]]
+    pred = model.predict(seq, cand)
+    df = pd.DataFrame({
+        "pos": [c[0] for c in cand],
+        "wt": [c[1] for c in cand],
+        "mut": [c[2] for c in cand],
+        "variant": [f"{c[1]}{c[0] + 1}{c[2]}" for c in cand],   # 1-based, e.g. A123V
+        "pred_gain": pred,
+        "model": model.name,
+    })
+    return df.sort_values("pred_gain", ascending=False).head(top).reset_index(drop=True)
+
+
+def retrospective_recovery(proposals: pd.DataFrame, known_variants: list[str], k: int = 20) -> dict:
+    """Blind-validation harness: does the top-k proposal set recover a published enhanced variant?
+
+    ``known_variants`` are 1-based strings like "A123V". Returns recovery flags per known variant and an
+    overall hit. With the Phase-1.5 DMS model this is the headline retrospective criterion; with the
+    baseline it merely demonstrates the harness runs (recovery is not expected from the placeholder).
+    """
+    topk = set(proposals.head(k)["variant"])
+    hits = {v: (v in topk) for v in known_variants}
+    return {"k": k, "model": proposals["model"].iloc[0] if len(proposals) else None,
+            "known": known_variants, "recovered": hits, "any_recovered": any(hits.values())}
+
+
+def run(enzyme: str, seq: str, model: VariantEffectModel | None = None, top: int = 20,
+        out_dir: str | Path = _OUT) -> pd.DataFrame:
+    model = model or BaselinePhysicoChemical()
+    props = propose_variants(seq, model, top=top)
+    out = Path(out_dir) / f"variant_proposals_{enzyme}.csv"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    props.to_csv(out, index=False)
+    return props
+
+
+# published enhancing single mutations identified by the Perry 2025 ISCro4 DMS (the known enhancers)
+KNOWN_ISCRO4_ENHANCERS = ["N322P", "H50K", "R278M"]
+
+
+def iscro4_dms_recovery(top: int = 20, out_dir: str | Path = _OUT) -> dict:
+    """Step 2.4 completion: feed the REAL Perry DMS model to the proposal framework and confirm it recovers
+    the known enhancing ISCro4 mutations in its top proposals. Honest framing: recovers KNOWN enhancers
+    (a catalogue feature), not a blind prediction. Returns the recovery report; writes the proposals CSV.
+    Empty/None when the Perry tables are absent."""
+    seq = iscro4_sequence()
+    if seq is None:
+        return {"available": False, "note": "Perry 2025 Table S1 (ISCro4 sequence) not present"}
+    props = run("ISCro4", seq, model=DMSVariantEffectModel(), top=top, out_dir=out_dir)
+    rec = retrospective_recovery(props, KNOWN_ISCRO4_ENHANCERS, k=top)
+    rec["available"] = True
+    rec["top_proposals"] = props.head(5)[["variant", "pred_gain"]].to_dict("records")
+    rec["framing"] = "recovers KNOWN enhancers from the measured DMS (catalogue feature); not a blind " \
+                     "sequence predictor and not a generative method (EVOLVEpro is the engine to wrap)."
+    return rec
+
+
+if __name__ == "__main__":  # pragma: no cover
+    # ISCro4 is 326 aa; without the protein sequence on hand we demo the harness on a short stub.
+    demo = "MSEQNKI" * 5
+    p = run("DEMO_stub", demo, top=10)
+    print(p.to_string(index=False))
+    print("\nNOTE: uses the labelled placeholder model; the DMS-trained predictor is a Phase-1.5 deliverable.")

pen_stack/bridge/__init__.py

@@ -0,0 +1 @@
+"""pen_stack.bridge - see PEN-STACK v3.0 program doc."""

pen_stack/bridge/activity.py

@@ -0,0 +1,52 @@
+"""Bridge-recombinase variant-effect, from the deep mutational scan (Phase 1.5, Step 1.5.4) - SECONDARY.
+
+A pluggable trainer over the Perry 2025 DMS (Table S3). Used retrospectively it RECOVERS KNOWN
+activity-enhancing mutants (N322P, H50K, R278M; see pen_stack/validate/paper4_real_validation.py),
+completing the Phase-2 Step-2.4 DMS variant-proposal feature.
+
+Scope, stated plainly: this is a useful catalogue feature that recovers KNOWN enhancers; it is NOT a novel
+variant-design method. For GENERATING new variants the established engine is EVOLVEpro - when PEN-STACK
+reaches generative variant design it should wrap EVOLVEpro rather than rebuild it. The 72-system ortholog
+screen (Table S1) carries no per-system activity label, so it supports only the descriptive characterisation
+in ortholog_screen.py (N ~72, exploratory). The headline of the phase is the off-target screening engine.
+"""
+from __future__ import annotations
+
+import pandas as pd
+
+
+def have_training_data(dms: pd.DataFrame, screen: pd.DataFrame) -> bool:
+    return (dms is not None and not dms.empty) or (screen is not None and not screen.empty)
+
+
+def train_variant_effect(dms_df: pd.DataFrame):
+    """Train a per-residue mutation -> activity model on the DMS. Returns None if no DMS available."""
+    if dms_df is None or dms_df.empty:
+        return None
+    import lightgbm as lgb
+    feat = pd.get_dummies(dms_df[["aa_position", "wt", "mut"]].astype(str))
+    return lgb.LGBMRegressor(n_estimators=400, learning_rate=0.03).fit(feat, dms_df["activity"])
+
+
+def train_ortholog_activity(screen_df: pd.DataFrame, embed_fn=None):
+    """Train ortholog -> human-cell activity on the 72-system screen. Returns None if absent.
+
+    N caveat is the caller's responsibility to report - the screen is ~72 systems.
+    """
+    if screen_df is None or screen_df.empty:
+        return None
+    import lightgbm as lgb
+    if embed_fn is not None:
+        X = embed_fn(screen_df["sequence"])
+    else:
+        X = pd.get_dummies(screen_df.get("target_core", pd.Series(dtype=str)).astype(str))
+    return lgb.LGBMRegressor(n_estimators=300, learning_rate=0.03).fit(X, screen_df["human_cell_activity"])
+
+
+def status() -> dict:
+    """Report whether the activity model can train (needs the Perry 2025 DMS / screen tables)."""
+    from pen_stack.bridge.ingest import load_dms, load_screen
+    dms, screen = load_dms(), load_screen()
+    return {"dms_rows": len(dms), "screen_rows": len(screen),
+            "trainable": have_training_data(dms, screen),
+            "note": "exploratory; DMS+screen are Perry 2025 supplementary (paywalled) - model trains when supplied"}