pen-stack 0.1.0__py3-none-any.whl

pen_stack/api/manifest.py

@@ -0,0 +1,160 @@
+"""Capability + scope manifests, the self-describing AI contract.
+
+The differentiator: PEN-STACK describes not only WHAT it can do (capability manifest) but, uniquely,
+machine-readably WHAT IT REFUSES TO ANSWER (scope manifest: the known-unknowns registry + the oracle scope
+cards). An external agent fetches both and ROUTES on them, instead of reading prose. Both are generated from the
+live registry / scope cards (never hand-written), versioned under the public API stability commitment.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+import pen_stack
+
+# the stable public tools (the 1.0 surface). Each is a thin contract over a validated entry point; none fabricate.
+_TOOLS = [
+    {"name": "verify_write", "summary": "legality + safety + calibrated confidence + immune profile for a design",
+     "input": "Design", "output": "Verdict", "entrypoint": "pen_stack.verify.verify", "fabricates": False},
+    {"name": "verify_proof", "summary": "repair-oriented proof object: legality / confidence / biosecurity as "
+                                        "separate axes with repair hints (never collapsed)",
+     "input": "Design", "output": "Proof", "entrypoint": "pen_stack.verify.proof.verify_proof",
+     "fabricates": False},
+    {"name": "safety_screen", "summary": "biosecurity / dual-use gate (clear/flag/escalate/refuse)",
+     "input": "Design", "output": "SafetyVerdict", "entrypoint": "pen_stack.safety.safety_gate",
+     "fabricates": False},
+    {"name": "generate_designs", "summary": "grounded candidate writing systems (verifier-as-discriminator)",
+     "input": "Goal | candidates", "output": "list[Design(candidate)]",
+     "entrypoint": "pen_stack.design.generate_designs", "fabricates": False},
+    {"name": "pareto_front", "summary": "non-dominated tradeoff frontier (incl. grounded immune-risk axis)",
+     "input": "list[Design]", "output": "list[Design(candidate)]",
+     "entrypoint": "pen_stack.design.pareto_front", "fabricates": False},
+    {"name": "predict_outcome", "summary": "calibrated, OOD-gated, phenotype-bounded write outcome",
+     "input": "Design + cell_state", "output": "Outcome(candidate)",
+     "entrypoint": "pen_stack.twin.predict_outcome", "fabricates": False},
+    {"name": "immune_profile", "summary": "per-axis immune-risk screen (never collapsed into one number)",
+     "input": "Design", "output": "ImmuneProfile", "entrypoint": "pen_stack.planner.immune_profile.immune_profile",
+     "fabricates": False},
+    {"name": "suggest_experiment", "summary": "active-learning next-experiment batch (EIG + immune-VOI)",
+     "input": "Designs + cell_state", "output": "ExperimentBatch",
+     "entrypoint": "pen_stack.active.select_batch", "fabricates": False},
+    {"name": "co_scientist_session", "summary": "drive the full loop: strategies + outcomes + experiments + safety",
+     "input": "Goal + cell_state", "output": "Session", "entrypoint": "pen_stack.agent.co_scientist.co_scientist_session",
+     "fabricates": False},
+    {"name": "run_loop", "summary": "one gated DBTL loop (autonomy Level 3, human in control)",
+     "input": "Goal + cell_state", "output": "LoopHistory", "entrypoint": "pen_stack.loop.run_loop",
+     "fabricates": False},
+    {"name": "challenge_evaluate", "summary": "score a submission on a held-out Genome-Writing Challenge round",
+     "input": "Submission + round_id", "output": "ChallengeResult",
+     "entrypoint": "benchmarks.genome_writing_challenge.harness.evaluate", "fabricates": False},
+    {"name": "recommend_writers", "summary": "rank writer families (KB-grounded primary) + candidate predicted "
+     "efficiency w/ conformal interval + auto-designed guide/att",
+     "input": "write request (write-type, cargo, cell type, optional target/donor seq)", "output": "WriterRanking",
+     "entrypoint": "pen_stack.atlas.writer_recommend.recommend_writers", "fabricates": False},
+    {"name": "nominate_offtargets", "summary": "genome-wide, per-mechanism off-target FINDER: "
+     "enumerates the off-target set over GRCh38 (Cas-OFFinder) and applies the correct mechanism per writer class "
+     "- nuclease cleavage (validated: CRISOT + mismatch-calibrated risk; chromatin accessibility is a validated "
+     "annotation surfaced when the accessibility track is mounted, not a re-ranker), integrase pseudo-attP (semi-validated), bridge "
+     "target-specificity (unvalidated), CAST guide + untargeted transposition (unvalidated), PASTE composition - "
+     "each with a truthful status label; nomination is NOT a clearance",
+     "input": "writer family + guide/target (enumerated genome-wide) or supplied candidate sites",
+     "output": "genome-wide ranked off-target candidates + per-mechanism status + calibrated risk + validation assay",
+     "entrypoint": "pen_stack.wgenome.offtarget_predict.nominate_offtargets", "fabricates": False},
+    {"name": "recommend_delivery", "summary": "cross-modality delivery recommender: rank vehicles by cargo-form + "
+     "safety<->efficacy + a grounded serotype->tissue tropism prior (approved therapies) + the learned FLIP-AAV "
+     "capsid-fitness capability; tropism is a known-unknown for novel capsids",
+     "input": "cargo form/size + target tissue (+ safety weight, in/ex-vivo)",
+     "output": "ranked vehicles + serotype tropism prior + capsid-fitness bench; never fabricates tropism",
+     "entrypoint": "pen_stack.planner.delivery_predict.recommend_delivery_plus", "fabricates": False},
+    {"name": "capsid_fitness", "summary": "learned AAV capsid packaging-fitness for a VP1 sequence (FLIP-AAV-"
+     "trained; beats a mutation-burden baseline on held-out splits); a CANDIDATE for the measured packaging axis, "
+     "NOT an in-vivo tropism claim",
+     "input": "AAV VP1 capsid amino-acid sequence", "output": "predicted packaging-fitness (candidate) or abstain",
+     "entrypoint": "pen_stack.planner.delivery_predict.capsid_fitness", "fabricates": False},
+    {"name": "validation_campaign", "summary": "the validation-campaign engine: the next batch of (cassette x "
+     "locus x cell type) expression measurements ordered by expected information gain, the calibrate_axis gate it "
+     "targets (the path to the program's first outcome-validated axis), and the active-vs-random result reported "
+     "verbatim; cloud-lab-executable, Level 3, experiments are candidates",
+     "input": "none", "output": "the expression-validation campaign (batch + target gate + acquisition result)",
+     "entrypoint": "pen_stack.active.campaign.design_campaign", "fabricates": False},
+    {"name": "cloudlab_submit", "summary": "safety-gated cloud-lab submission: the biosecurity gate runs BEFORE "
+     "submission, a flagged design returns a structured refusal (no protocol emitted), a cleared design returns a "
+     "mock/dry-run job receipt; Level 3, human in control",
+     "input": "a design (+ optional experiment)", "output": "a mock job receipt or a structured biosecurity "
+     "refusal", "entrypoint": "pen_stack.build.cloudlab.submit_gated", "fabricates": False},
+    {"name": "writespec_parse", "summary": "parse a plain-language genome-writing request into a typed, "
+     "ontology-backed WriteSpec (an SBOL3 profile): per-field provenance (explicit/inferred/user/unresolved), "
+     "assumptions, clarifying questions on underspecification, and a feasibility verdict (reachability + "
+     "deliverability + legality); a REQUEST not a claim, never fabricates intent",
+     "input": "prose request (+ optional structured overrides)", "output": "typed WriteSpec + clarifications + "
+     "feasibility; unresolved stays null", "entrypoint": "pen_stack.spec.service.parse_request", "fabricates": False},
+    {"name": "oracle_query", "summary": "query the oracle mesh under one contract: per-oracle execution + latency "
+     "+ live status + PUBLISHED reliability (verbatim from public benchmarks, cited) + disagreement-to-interval; "
+     "or a CANDIDATE protein-ligand binding affinity (Boltz-2 head) with native uncertainty, cache-or-abstain",
+     "input": "oracle name, or {protein_seq, ligand_smiles, pair_type}", "output": "oracle status + reliability, "
+     "or a candidate affinity (binder probability + value) or abstain; protein-protein/protein-DNA flagged OOD",
+     "entrypoint": "pen_stack.oracles.affinity.predict_affinity", "fabricates": False},
+    {"name": "chat_answer", "summary": "the grounded conversational co-scientist: a provenance-"
+     "partitioned 4-lane agent (design/explain/meta = engine-grounded with the guard ON; general = retrieval-"
+     "grounded over a provenance-tagged corpus under citation-or-silence, abstaining below a retrieval-confidence "
+     "threshold). The LLM narrates and is swappable; it NEVER originates a claim or a number. Measured by the "
+     "chat_routing / chat_grounding / chat_safety benchmarks (routing-safety ~0, citation coverage 1.0, "
+     "false-grounding ~0)",
+     "input": "{message, history?, allow_llm?}", "output": "{reply, mode (lane), provenance, grounded, sources?, "
+     "backend}; general answers are 'literature-cited' or 'abstained', never a PEN-STACK-computed result",
+     "entrypoint": "pen_stack.web.llm.grounded_reply", "fabricates": False},
+]
+
+_POLICY = ("outputs outside scope are returned as `out_of_scope` (known-unknown) or `extrapolating` (OOD) and are "
+           "NEVER asserted; every number is tool-sourced (no fabrication); contracts are versioned + "
+           "deprecation-policed under the public API stability commitment (docs/STABILITY.md).")
+
+
+def capability_manifest() -> dict[str, Any]:
+    """Machine-readable: WHAT PEN-STACK can do. An agent routes on this, not on prose."""
+    return {
+        "name": "pen-stack", "version": pen_stack.__version__, "stability": "stable",
+        "contract_version": "0.1.0",
+        "guarantees": ["rule-grounded legality", "calibrated confidence", "explicit scope / known-unknowns",
+                       "biosecurity safety gate", "no fabrication"],
+        "tools": _TOOLS,
+        "oracles": _oracle_summary(),
+        "surfaces": {"rest": "/capabilities, /scope, /oracles, /verify, /generate, /predict, /immune, /safety, "
+                             "/suggest, /session, /openapi.json", "mcp": "pen_stack.agent.mcp_server",
+                     "challenge": "benchmarks/genome_writing_challenge"},
+    }
+
+
+def _oracle_summary() -> dict:
+    """Live-oracle roll-up: which foundation models execute live, their latency class, what is held/deferred."""
+    from pen_stack.oracles.status import summary
+    return summary()
+
+
+def _known_unknowns() -> list[dict]:
+    """The known-unknowns as PUBLIC data (id/title/requires/why), internal matcher fields are not exposed."""
+    from pen_stack.agent.scope import load_registry
+    return [{"id": e["id"], "title": e["title"], "requires": e.get("requires"),
+             "why": " ".join((e.get("why") or "").split())} for e in load_registry()]
+
+
+def _oracle_scope_cards() -> list[dict]:
+    """Each wrapped model's in-distribution envelope (what it is valid for, and what it is NOT)."""
+    from pen_stack.oracles.cache import load_scope_cards
+    out = []
+    for model, c in load_scope_cards().items():
+        out.append({"model": model, "family": c.get("family"), "version": c.get("version"),
+                    "output_kind": c.get("output_kind"),
+                    "valid_for": " ".join((c.get("valid_for") or "").split()),
+                    "not_valid_for": " ".join((c.get("not_valid_for") or "").split()),
+                    "generalizes_to_unseen_loci": c.get("generalizes_to_unseen_loci")})
+    return out
+
+
+def scope_manifest() -> dict[str, Any]:
+    """Machine-readable: WHAT PEN-STACK REFUSES TO ANSWER. The contract that makes depending on it safe."""
+    return {
+        "name": "pen-stack", "version": pen_stack.__version__, "contract_version": "0.1.0",
+        "known_unknowns": _known_unknowns(),
+        "oracle_scope_cards": _oracle_scope_cards(),
+        "policy": _POLICY,
+    }

pen_stack/atlas/__init__.py

@@ -0,0 +1 @@
+"""pen_stack.atlas - see the PEN-STACK program doc."""

pen_stack/atlas/build_wtkb.py

@@ -0,0 +1,80 @@
+"""Build the Writer-Targeting Knowledge Base (WT-KB).
+
+Reads the curated YAML (one block per writer family), validates every row against the
+``WriterEntry`` schema (which enforces the sourcing rule: >=1 DOI per row), and emits both a
+parquet (for the pipeline) and a human-readable markdown table (for literature cross-check).
+
+Usage:
+    python -m pen_stack.atlas.build_wtkb --curated configs/wtkb_curated.yaml \
+        --out pen_stack/atlas/wtkb.parquet --md docs/wtkb.md
+"""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+from pen_stack.atlas.schema import WriterEntry
+
+
+def build(curated_yaml: str, out_parquet: str | None = None, out_md: str | None = None) -> pd.DataFrame:
+    curated = yaml.safe_load(Path(curated_yaml).read_text(encoding="utf-8"))
+    rows = []
+    for key, block in curated.items():
+        entry = WriterEntry(**block) # validates (raises on missing DOI / bad enum)
+        d = entry.model_dump()
+        d["_key"] = key
+        rows.append(d)
+    df = pd.DataFrame(rows)
+    # stable column order, _key first
+    cols = ["_key"] + [c for c in df.columns if c != "_key"]
+    df = df[cols]
+    if out_parquet:
+        Path(out_parquet).parent.mkdir(parents=True, exist_ok=True)
+        df.to_parquet(out_parquet, index=False)
+    if out_md:
+        Path(out_md).parent.mkdir(parents=True, exist_ok=True)
+        Path(out_md).write_text(_to_markdown(df), encoding="utf-8")
+    return df
+
+
+def _to_markdown(df: pd.DataFrame) -> str:
+    lines = [
+        "# Writer-Targeting Knowledge Base (WT-KB)",
+        "",
+        f"_Generated from `configs/wtkb_curated.yaml` - {len(df)} writer families. "
+        "Every row is schema-validated and carries >=1 DOI (sourcing rule)._",
+        "",
+        "| Family | Representative | Mechanism | Modality | Target site | Tier | Confidence | DOIs |",
+        "|---|---|---|---|---|---|---|---|",
+    ]
+    for _, r in df.iterrows():
+        dois = "; ".join(r["key_dois"])
+        lines.append(
+            f"| {r['family']} | {r['representative_system']} | {r['mechanism_bucket']} | "
+            f"{r['targeting_modality']} | {r['target_site_spec']} | {r['reachability_tier']} | "
+            f"{r['confidence']} | {dois} |"
+        )
+    lines += ["", "## Reachability constraints (per family)", ""]
+    for _, r in df.iterrows():
+        lines.append(f"- **{r['family']}** ({r['reachability_tier']}): {r['reachability_constraints']}")
+    return "\n".join(lines) + "\n"
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--curated", default="configs/wtkb_curated.yaml")
+    ap.add_argument("--out", default="pen_stack/atlas/wtkb.parquet")
+    ap.add_argument("--md", default="docs/wtkb.md")
+    a = ap.parse_args()
+    df = build(a.curated, a.out, a.md)
+    print(f"WT-KB built: {len(df)} families -> {a.out}")
+    tiers = df["reachability_tier"].value_counts().to_dict()
+    print(f"tiers: {tiers}")
+    print(f"fully-specified families: {len(df)} (target >=6)")
+
+
+if __name__ == "__main__":
+    main()

pen_stack/atlas/crosslink.py

@@ -0,0 +1,179 @@
+"""Cross-link the Writer Atlas <-> the Writable Genome.
+
+The integration that makes PEN-STACK more than a catalogue: bidirectional queries between writers (the
+atlas, 33k systems by family) and loci (the Writable Genome, 3M bins x cell type with a
+``reachable_tier1`` annotation + a decomposable ``writability`` score).
+
+- ``loci_for_writer(family, ct)`` -> loci that family can reach, ranked by writability.
+- ``writers_for_locus(chrom, bin)`` -> atlas systems whose family reaches that locus, with readiness.
+- ``loci_for_gene(gene, ct)`` -> writable bins overlapping a gene (forward query helper).
+
+Scope: reachability is released at the *locus* level - the Tier-1
+reprogrammable families (bridge_IS110 / Cas9 / Cas12a) are near-universal at 1 kb, so the cross-link's
+discriminating signal is the *writability ranking* and the *family -> atlas-system* join (each carrying
+therapeutic readiness). Per-site reachability (does a specific bridge core exist here?) is Planner work.
+
+Inputs : atlas_<ct>.parquet (chrom, bin, safety, p_durable, reachable_tier1, writability),
+         the atlas.parquet, gene_coords.parquet.
+Outputs: out/crosslink_cache_<ct>.parquet (per-family reachable-loci summary).
+"""
+from __future__ import annotations
+
+import os
+from functools import lru_cache
+from pathlib import Path
+
+import pandas as pd
+
+_ROOT = Path(__file__).resolve().parents[2]
+_FINAL = _ROOT.parent # parent (Final_Part) directory
+_ATLAS = _ROOT / "pen_stack" / "atlas" / "atlas.parquet"
+_OUT = _ROOT / "out"
+BIN_BP = 1000
+
+# writability atlas can live in a few places (fetched-not-committed). First match wins.
+# PEN_ATLAS_DIR (also used by the UI) is honoured first so every cross-link-backed feature - the Write
+# Planner, the agent, and the RAG numeric route - finds the same atlas the UI does in any deployment.
+def _writability_search() -> list[Path]:
+    bases: list[Path] = []
+    env = os.environ.get("PEN_ATLAS_DIR")
+    if env:
+        bases.append(Path(env))
+    bases += [_ROOT / "data" / "out", _FINAL / "phase_1" / "out"]
+    return bases
+
+
+def writability_path(ct: str) -> Path:
+    bases = _writability_search()
+    for base in bases:
+        p = base / f"atlas_{ct}.parquet"
+        if p.exists():
+            return p
+    raise FileNotFoundError(f"atlas_{ct}.parquet not found in {[str(b) for b in bases]}")
+
+
+# Cell types whose per-ct genotoxic-safety model is a partial / degraded computation (the atlas ships them as
+# "partial" coverage). Their safety column can fail UNSAFE, claim a locus is safe where a full-coverage cell type
+# correctly flags it (e.g. HSPC scoring bins beside the LMO2 oncogene as safe). Genotoxic proximity is a GENOMIC,
+# cell-type-independent property, so for these we take a CONSERVATIVE floor from the full-coverage atlases.
+_PARTIAL_SAFETY_CT = {"hspc"}
+_FULL_COVERAGE_CT = ("k562", "hepg2")
+
+
+@lru_cache(maxsize=1)
+def _full_coverage_safety_floor() -> "pd.Series | None":
+    """min(safety) over the full-coverage atlases per (chrom, bin), the reliable, cell-type-independent genotoxic
+    floor used to correct a partial-coverage cell type's safety. None if no full-coverage atlas is present."""
+    frames = []
+    for r in _FULL_COVERAGE_CT:
+        try:
+            frames.append(pd.read_parquet(writability_path(r))[["chrom", "bin", "safety"]])
+        except FileNotFoundError:
+            continue
+    if not frames:
+        return None
+    allf = pd.concat(frames, ignore_index=True)
+    return allf.groupby(["chrom", "bin"])["safety"].min()
+
+
+@lru_cache(maxsize=4)
+def load_writability(ct: str) -> pd.DataFrame:
+    df = pd.read_parquet(writability_path(ct))
+    if ct.lower() in _PARTIAL_SAFETY_CT:
+        floor = _full_coverage_safety_floor()
+        if floor is not None:
+            df = df.merge(floor.rename("_ref_safety"), on=["chrom", "bin"], how="left")
+            df["safety_raw"] = df["safety"]
+            # fail-SAFE: never claim safer than the reliable full-coverage reference at the same genomic bin
+            df["safety"] = df[["safety", "_ref_safety"]].min(axis=1).fillna(df["safety"])
+            df["writability"] = 0.5 * df["safety"] + 0.5 * df["p_durable"]
+            df["safety_coverage"] = "partial: conservative floor from full-coverage atlases (genomic genotoxic signal)"
+            df = df.drop(columns=["_ref_safety"])
+    # Reject-known-bad genotoxicity blocklist: documented clinical insertional-oncogenesis loci are flagged
+    # unsafe (safety -> 0) by rule, overriding the soft model (closes the BMI1/SETBP1/MN1 false negatives).
+    from pen_stack.wgenome.genotoxic_blocklist import apply_genotoxic_blocklist
+    df = apply_genotoxic_blocklist(df)
+    df["_reach"] = df["reachable_tier1"].fillna("").str.split(";")
+    return df
+
+
+@lru_cache(maxsize=1)
+def load_writer_atlas() -> pd.DataFrame:
+    return pd.read_parquet(_ATLAS)
+
+
+def reachable_families(ct: str) -> set[str]:
+    """The writer families annotated as Tier-1 reachable in the atlas for this cell type."""
+    df = load_writability(ct)
+    fams: set[str] = set()
+    for r in df["reachable_tier1"].dropna().unique():
+        fams.update(x for x in str(r).split(";") if x)
+    return fams
+
+
+def loci_for_writer(family: str, ct: str = "k562", top: int = 20) -> pd.DataFrame:
+    """Top-writability loci reachable by a writer family (genomic coords + writability components)."""
+    df = load_writability(ct)
+    mask = df["_reach"].apply(lambda fams: family in fams)
+    hit = df.loc[mask].nlargest(top, "writability").copy()
+    hit["chrom_start"] = hit["bin"] * BIN_BP
+    return hit[["chrom", "bin", "chrom_start", "safety", "p_durable", "writability", "reachable_tier1"]]
+
+
+def writers_for_locus(chrom: str, bin_idx: int, ct: str = "k562") -> pd.DataFrame:
+    """Atlas systems whose family reaches a locus, with therapeutic readiness (if scored)."""
+    df = load_writability(ct)
+    row = df[(df["chrom"] == chrom) & (df["bin"] == bin_idx)]
+    if row.empty:
+        return pd.DataFrame()
+    fams = {x for x in str(row.iloc[0]["reachable_tier1"]).split(";") if x}
+    atlas = load_writer_atlas()
+    cols = [c for c in ["representative_system", "family", "confidence", "deliv_class",
+                        "readiness", "cargo_capacity_bp", "reachability_tier"] if c in atlas.columns]
+    out = atlas[atlas["family"].isin(fams)][cols].copy()
+    out["locus_writability"] = float(row.iloc[0]["writability"])
+    return out
+
+
+def loci_for_gene(gene: str, ct: str = "k562", gene_coords: str | Path | None = None) -> pd.DataFrame:
+    """Writable bins overlapping a gene body (forward query helper)."""
+    from pen_stack.planner.optimize import gene_coords_path, resolve_gene
+    gc_path = Path(gene_coords) if gene_coords else gene_coords_path()
+    gc = pd.read_parquet(gc_path)
+    g = gc[gc["gene"] == resolve_gene(gene)] # resolve safe-harbour nicknames (AAVS1 -> PPP1R12C)
+    if g.empty:
+        return pd.DataFrame()
+    r = g.iloc[0]
+    df = load_writability(ct)
+    lo, hi = int(r["start"]) // BIN_BP, int(r["end"]) // BIN_BP
+    return df[(df["chrom"] == r["chrom"]) & (df["bin"].between(lo, hi))].sort_values(
+        "writability", ascending=False)
+
+
+def build_crosslink_cache(ct: str = "k562", out: str | Path | None = None) -> pd.DataFrame:
+    """Per-family reachable-loci summary (count + median writability + top bin), cached."""
+    df = load_writability(ct)
+    rows = []
+    for fam in sorted(reachable_families(ct)):
+        sub = df[df["_reach"].apply(lambda fams, f=fam: f in fams)]
+        top = sub.nlargest(1, "writability")
+        rows.append({
+            "family": fam, "cell_type": ct, "n_reachable_loci": len(sub),
+            "median_writability": round(float(sub["writability"].median()), 4),
+            "top_chrom": top.iloc[0]["chrom"], "top_bin": int(top.iloc[0]["bin"]),
+            "top_writability": round(float(top.iloc[0]["writability"]), 4),
+        })
+    cache = pd.DataFrame(rows)
+    out = Path(out) if out else _OUT / f"crosslink_cache_{ct}.parquet"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    cache.to_parquet(out, index=False)
+    return cache
+
+
+if __name__ == "__main__": # pragma: no cover
+    for ct in ("k562", "hepg2", "hspc"):
+        try:
+            c = build_crosslink_cache(ct)
+            print(f"[{ct}] crosslink cache:\n{c.to_string(index=False)}\n")
+        except Exception as e: # noqa: BLE001 - a missing/partial cell-type atlas is non-fatal
+            print(f"[{ct}] skip: {e}")

pen_stack/atlas/expand.py

@@ -0,0 +1,190 @@
+"""Expand the Writer Atlas across families.
+
+Grow the curated 8-family core into a comprehensive cross-family catalogue: ingest ortholog
+sets at scale (the IS110/IS1111 superfamily, CAST, large serine integrases, Cas12a, TnpB/Fanzor) from
+UniProt, place every entry on the WT-KB targeting axes by *inheriting* family-level metadata from the
+``wtkb.parquet`` (single source of truth - the classifier/scorer must not re-derive it), and
+tag each row with an explicit ``confidence`` (measured / inferred / predicted) and provenance.
+
+Heavy per-ortholog featurisation (ESM embeddings for mechanism classification at scale) runs
+in Docker on the GPU; this module only assembles the *catalogue* metadata (lightweight, network-bound).
+
+Inputs : configs/atlas_families.yaml, pen_stack/atlas/wtkb.parquet, UniProt REST.
+Outputs: pen_stack/atlas/atlas.parquet (one row per system), cached TSVs under data/external/atlas/.
+"""
+from __future__ import annotations
+
+import time
+import urllib.parse
+import urllib.request
+from io import StringIO
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+_ROOT = Path(__file__).resolve().parents[2]
+_CFG = _ROOT / "configs" / "atlas_families.yaml"
+_WTKB = _ROOT / "pen_stack" / "atlas" / "wtkb.parquet"
+_CACHE = _ROOT / "data" / "external" / "atlas"
+_OUT = _ROOT / "pen_stack" / "atlas" / "atlas.parquet"
+
+_UNIPROT_STREAM = "https://rest.uniprot.org/uniprotkb/stream"
+
+# WT-KB family-level fields every atlas row inherits (so targeting metadata has ONE source).
+_INHERIT = [
+    "mechanism_bucket", "targeting_modality", "target_site_spec", "guide_architecture",
+    "cargo_mechanism", "cargo_capacity_bp", "dsb_free", "reachability_tier",
+    "reachability_constraints",
+]
+
+
+def load_config(path: str | Path = _CFG) -> dict:
+    return yaml.safe_load(Path(path).read_text(encoding="utf-8"))
+
+
+def fetch_uniprot(query: str, fields: str, cache: Path, timeout: int = 120,
+                  retries: int = 3) -> pd.DataFrame:
+    """Stream a UniProt query to TSV (cached). Returns the raw per-accession metadata frame."""
+    cache.parent.mkdir(parents=True, exist_ok=True)
+    if cache.exists():
+        return pd.read_csv(cache, sep="\t", dtype=str)
+    params = {"query": query, "format": "tsv", "fields": fields}
+    url = _UNIPROT_STREAM + "?" + urllib.parse.urlencode(params)
+    last = None
+    for attempt in range(retries):
+        try:
+            with urllib.request.urlopen(url, timeout=timeout) as r:
+                text = r.read().decode("utf-8")
+            df = pd.read_csv(StringIO(text), sep="\t", dtype=str)
+            cache.write_text(text, encoding="utf-8")
+            return df
+        except Exception as e: # noqa: BLE001 - network is best-effort; surfaced after retries
+            last = e
+            time.sleep(2 * (attempt + 1))
+    raise RuntimeError(f"UniProt fetch failed for query {query!r}: {last}")
+
+
+def _orthologs_for_family(fam_key: str, fam: dict, fields: str, wtkb_row: pd.Series,
+                          cache_dir: Path) -> pd.DataFrame:
+    """One ortholog table for a family, inheriting WT-KB targeting metadata by family."""
+    cache = cache_dir / f"{fam_key}.tsv"
+    raw = fetch_uniprot(fam["query"], fields, cache)
+    cap = int(fam.get("cap", len(raw)))
+    raw = raw.head(cap).copy()
+
+    # UniProt TSV column names (from the requested fields)
+    acc_col = "Entry"
+    org_col = "Organism"
+    len_col = "Length"
+    df = pd.DataFrame({
+        "representative_system": raw.get(acc_col),
+        "uniprot": raw.get(acc_col),
+        "organism": raw.get(org_col),
+        "length_aa": pd.to_numeric(raw.get(len_col), errors="coerce"),
+    })
+    df["family"] = fam["wtkb_family"]
+    df["pfam_signature"] = [list(fam["pfam_signature"])] * len(df)
+    df["confidence"] = fam.get("default_confidence", "predicted")
+    df["human_cell_activity"] = "not measured (sequence homolog)"
+    df["key_dois"] = [[fam["discovery_doi"]]] * len(df)
+    df["entry_kind"] = "ortholog"
+    # inherit targeting metadata from the WT-KB family row (single source of truth)
+    for col in _INHERIT:
+        df[col] = wtkb_row.get(col)
+    return df
+
+
+def _curated_rows(cfg: dict, wtkb: pd.DataFrame) -> pd.DataFrame:
+    """Named, characterised systems: the 8 WT-KB families themselves + extra reps from config."""
+    rows = []
+    # (a) the WT-KB curated core - measured/inferred, full targeting spec
+    for _, w in wtkb.iterrows():
+        rows.append({
+            "representative_system": w["representative_system"],
+            "uniprot": w.get("uniprot"),
+            "organism": None,
+            "length_aa": w.get("length_aa"),
+            "family": w["family"],
+            "pfam_signature": list(w["pfam_signature"]) if w.get("pfam_signature") is not None else [],
+            "confidence": w.get("confidence", "measured"),
+            "human_cell_activity": w.get("human_cell_activity"),
+            "key_dois": list(w["key_dois"]) if w.get("key_dois") is not None else [],
+            "entry_kind": "curated_core",
+            **{c: w.get(c) for c in _INHERIT},
+        })
+    # (b) extra curated representatives (named systems w/o a clean single-Pfam query)
+    wt_by_fam = wtkb.set_index("family")
+    for rep in cfg.get("curated_representatives", []):
+        fam = rep["family"]
+        wrow = wt_by_fam.loc[fam] if fam in wt_by_fam.index else pd.Series(dtype=object)
+        if isinstance(wrow, pd.DataFrame):
+            wrow = wrow.iloc[0]
+        rows.append({
+            "representative_system": rep["representative_system"],
+            "uniprot": rep.get("uniprot"),
+            "organism": None,
+            "length_aa": rep.get("length_aa") or (wrow.get("length_aa") if len(wrow) else None),
+            "family": fam,
+            "pfam_signature": list(wrow.get("pfam_signature")) if len(wrow) and wrow.get("pfam_signature") is not None else [],
+            "confidence": rep.get("confidence", "inferred"),
+            "human_cell_activity": rep.get("human_cell_activity"),
+            "key_dois": list(rep.get("key_dois", [])),
+            "entry_kind": "curated_rep",
+            **{c: (wrow.get(c) if len(wrow) else None) for c in _INHERIT},
+        })
+    return pd.DataFrame(rows)
+
+
+def confidence_tag(row: pd.Series) -> str:
+    """measured (human-cell data) > inferred (characterised, non-human) > predicted (homolog only)."""
+    c = row.get("confidence")
+    if c in {"measured", "inferred", "predicted"}:
+        return c
+    hca = (row.get("human_cell_activity") or "").lower()
+    if "human cell" in hca and "not measured" not in hca:
+        return "measured"
+    return "predicted"
+
+
+def build_atlas(cfg_path: str | Path = _CFG, wtkb_path: str | Path = _WTKB,
+                out: str | Path = _OUT, cache_dir: str | Path = _CACHE,
+                offline_ok: bool = False) -> pd.DataFrame:
+    cfg = load_config(cfg_path)
+    fields = cfg["defaults"]["uniprot_fields"]
+    wtkb = pd.read_parquet(wtkb_path)
+    wt_by_fam = wtkb.drop_duplicates("family").set_index("family")
+    cache_dir = Path(cache_dir)
+
+    tables: list[pd.DataFrame] = [_curated_rows(cfg, wtkb)]
+    for fam_key, fam in cfg["families"].items():
+        wrow = wt_by_fam.loc[fam["wtkb_family"]] if fam["wtkb_family"] in wt_by_fam.index else pd.Series(dtype=object)
+        try:
+            tables.append(_orthologs_for_family(fam_key, fam, fields, wrow, cache_dir))
+        except Exception as e: # noqa: BLE001
+            if offline_ok:
+                print(f"[expand] skip {fam_key} (offline_ok): {e}")
+                continue
+            raise
+
+    atlas = pd.concat(tables, ignore_index=True)
+    # named curated rows win over a homolog row for the same accession. Dedup ONLY among rows that
+    # carry a UniProt id - rows without one (seekRNA, PASTE, ShCAST, Bxb1, ISPpu10) are all distinct
+    # systems and must never collapse together (pandas treats every NaN as equal under drop_duplicates).
+    atlas["_pri"] = atlas["entry_kind"].map({"curated_core": 0, "curated_rep": 1, "ortholog": 2})
+    has_acc = atlas["uniprot"].notna() & (atlas["uniprot"].astype(str).str.strip() != "")
+    with_acc = (atlas[has_acc].sort_values("_pri")
+                .drop_duplicates(subset=["uniprot"], keep="first"))
+    atlas = pd.concat([with_acc, atlas[~has_acc]], ignore_index=True).drop(columns="_pri")
+    atlas["confidence"] = atlas.apply(confidence_tag, axis=1)
+    atlas = atlas.reset_index(drop=True)
+
+    Path(out).parent.mkdir(parents=True, exist_ok=True)
+    atlas.to_parquet(out, index=False)
+    return atlas
+
+
+if __name__ == "__main__": # pragma: no cover
+    a = build_atlas()
+    print(f"atlas rows: {len(a):,}")
+    print(a.groupby(["family", "confidence"]).size())