pen-stack 0.1.0__py3-none-any.whl

pen_stack/spec/resolvers/locus.py

@@ -0,0 +1,29 @@
+"""Locus resolver: free text (gene / safe-harbour) -> GRCh38 chromosome, reusing the atlas resolver.
+
+Wraps :func:`pen_stack.planner.optimize.gene_region` (with the safe-harbour-aware ``resolve_gene``). An
+unresolvable term stays null. The coordinate granularity is the chromosome the gene body lies on (the full
+base-range comes from the atlas when a downstream reachability query is run).
+"""
+from __future__ import annotations
+
+from pen_stack.spec.writespec import Resolved
+
+
+def resolve_locus(text: str) -> Resolved:
+    """Resolve a gene / safe-harbour locus to its GRCh38 chromosome; unrecognized stays null."""
+    if not text:
+        return Resolved(text=text, ontology="GRCh38", note="empty")
+    sym = text.strip().upper()
+    reg = None
+    try:
+        from pen_stack.planner.optimize import gene_region, resolve_gene
+        reg = gene_region(resolve_gene(sym))  # -> (chrom, start, end) | None
+    except Exception: # noqa: BLE001
+        reg = None
+    if not reg:
+        return Resolved(text=text, ontology="GRCh38", confidence=None, note="unresolved locus")
+    chrom, start, end = reg
+    chrom = chrom if str(chrom).startswith("chr") else f"chr{chrom}"
+    cid = f"{chrom}:{int(start)}-{int(end)}"
+    return Resolved(text=text, id=cid, label=cid, ontology="GRCh38", confidence=0.85,
+                    note="GRCh38 gene-body region (+/- flank) from the atlas")

pen_stack/spec/resolvers/phenotype.py

@@ -0,0 +1,37 @@
+"""Phenotype / disease-goal resolver: free text -> MONDO. Ids verified against EBI OLS."""
+from __future__ import annotations
+
+from pen_stack.spec.writespec import Resolved
+
+# (MONDO id, canonical label) - verified live before commit
+_PHENO: dict[str, tuple[str, str]] = {
+    "sickle cell disease": ("MONDO:0011382", "sickle cell disease"),
+    "sickle cell": ("MONDO:0011382", "sickle cell disease"), "scd": ("MONDO:0011382", "sickle cell disease"),
+    "beta thalassemia": ("MONDO:0019402", "beta thalassemia"),
+    "beta-thalassemia": ("MONDO:0019402", "beta thalassemia"),
+    "duchenne muscular dystrophy": ("MONDO:0010679", "Duchenne muscular dystrophy"),
+    "duchenne": ("MONDO:0010679", "Duchenne muscular dystrophy"), "dmd": ("MONDO:0010679", "Duchenne muscular dystrophy"),
+    "cystic fibrosis": ("MONDO:0009061", "cystic fibrosis"), "cf": ("MONDO:0009061", "cystic fibrosis"),
+    "rett syndrome": ("MONDO:0010726", "Rett syndrome"), "rett": ("MONDO:0010726", "Rett syndrome"),
+    "hemophilia b": ("MONDO:0010604", "hemophilia B"),
+    "hemophilia a": ("MONDO:0010602", "hemophilia A"),
+    "leber congenital amaurosis 10": ("MONDO:0012723", "Leber congenital amaurosis 10"),
+    "lca10": ("MONDO:0012723", "Leber congenital amaurosis 10"),
+}
+
+
+def resolve_phenotype(text: str) -> Resolved:
+    """Resolve a disease / phenotype goal to MONDO; unrecognized stays null (never invented)."""
+    if not text:
+        return Resolved(text=text, ontology="MONDO", note="empty")
+    key = text.strip().lower()
+    hit = _PHENO.get(key)
+    if hit is None:
+        for k, v in _PHENO.items():
+            if k in key or key in k:
+                hit = v
+                break
+    if hit is None:
+        return Resolved(text=text, ontology="MONDO", confidence=None, note="unresolved phenotype term")
+    mid, label = hit
+    return Resolved(text=text, id=mid, label=label, ontology="MONDO", confidence=0.9)

pen_stack/spec/satisfy.py

@@ -0,0 +1,114 @@
+"""WriteSpec feasibility / satisfiability check.
+
+Given a :class:`WriteRequest`, test the NECESSARY conditions a write must meet, by wrapping the existing stages:
+
+  * reachability  - is the target locus / gene reachable by a writer in the writable-genome atlas?
+  * deliverability - does any vehicle fit the cargo size (the delivery recommender)?
+  * legality      - does the design pass the rule set (the repair-oriented proof object)?
+
+Returns ``feasible`` or ``infeasible`` plus the NAMED blocking constraint(s) and repair hints (from the rule-set
+proof and a delivery suggestion), so an agent can repair the spec and re-check. A check whose backend is absent is
+reported ``indeterminate`` and does not, by itself, block (it is surfaced, never silently passed). Feasibility is
+necessary-conditions only: it rules out unreachable / undeliverable / illegal, NOT whether the write will WORK
+(that is the downstream stages' calibrated, still-uncertain prediction).
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from pen_stack.spec.writespec import WriteRequest
+
+
+class SatisfyResult(BaseModel):
+    feasible: bool
+    checks: dict[str, Any] = Field(default_factory=dict)     # reachability / deliverability / legality verdicts
+    blocking: list[dict] = Field(default_factory=list)       # named blocking constraints
+    repairs: list[dict] = Field(default_factory=list)        # repair hints
+    note: str | None = None
+
+
+def _reachability(spec: WriteRequest) -> dict:
+    gene = spec.target.gene.id if (spec.target.gene and spec.target.gene.id) else None
+    if spec.target.kind == "att_site":
+        return {"ok": True, "status": "att/landing site supplied; reachability is by the installed att",
+                "determinable": True}
+    if gene is None:
+        return {"ok": None, "status": "no resolved target gene/locus; supply one for a reachability check",
+                "determinable": False}
+    try:
+        from pen_stack.atlas.crosslink import loci_for_gene
+        df = loci_for_gene(gene, ct="k562")
+        n = 0 if df is None else len(df)
+        if n > 0:
+            return {"ok": True, "status": f"{n} writable bins overlap {gene} in the atlas", "determinable": True}
+        return {"ok": False, "status": f"no writable bin overlaps {gene} in the atlas (cross-check the locus)",
+                "determinable": True}
+    except Exception as e: # noqa: BLE001
+        return {"ok": None, "status": f"atlas reachability unavailable ({type(e).__name__})", "determinable": False}
+
+
+def _deliverability(spec: WriteRequest) -> dict:
+    bp = spec.constraints.max_cargo_bp or sum((c.length_bp or 0) for c in spec.cargo) or None
+    if bp is None:
+        return {"ok": None, "status": "no cargo size given; supply bp/kb for a deliverability check",
+                "determinable": False}
+    try:
+        from pen_stack.planner.delivery_predict import recommend_delivery_plus
+        rec = recommend_delivery_plus("DNA", bp, None)
+        ranked = rec.get("ranked") or []   # the recommender already filters to vehicles that fit the cargo
+        excluded = rec.get("excluded") or []
+        if ranked:
+            top = ranked[0].get("vehicle") or ranked[0].get("name")
+            return {"ok": True, "status": f"{len(ranked)} vehicle(s) fit {bp} bp (top: {top}; "
+                    f"{len(excluded)} excluded by capacity)", "determinable": True, "top": top}
+        return {"ok": False, "status": f"no vehicle in the palette fits {bp} bp", "determinable": True}
+    except Exception: # noqa: BLE001
+        if bp <= 8000:
+            return {"ok": True, "status": f"{bp} bp is within typical single-vector capacity (<= ~8 kb)",
+                    "determinable": True}
+        return {"ok": False, "status": f"{bp} bp exceeds typical single-vector capacity (~8 kb); needs a "
+                "dual/high-capacity strategy", "determinable": True}
+
+
+def _legality(spec: WriteRequest) -> dict:
+    try:
+        from pen_stack.verify.proof import verify_proof
+        proof = verify_proof(spec.to_legacy_design())
+        ax = next((a for a in proof.axes if a.axis == "legality"), None)
+        if ax is None:
+            return {"ok": None, "status": "no legality axis returned", "determinable": False}
+        ok = ax.status in ("pass", "clear")
+        out = {"ok": ok, "status": ax.status, "determinable": True}
+        if not ok:
+            out["violated"] = [v for v in (ax.violated or [])]
+            if ax.repair_hint:
+                out["repair_hint"] = ax.repair_hint
+        return out
+    except Exception as e: # noqa: BLE001
+        return {"ok": None, "status": f"legality check unavailable ({type(e).__name__})", "determinable": False}
+
+
+def check_satisfiable(spec: WriteRequest) -> SatisfyResult:
+    """Run the reachability + deliverability + legality necessary-condition checks; name blocking constraints."""
+    checks = {"reachability": _reachability(spec), "deliverability": _deliverability(spec),
+              "legality": _legality(spec)}
+    blocking: list[dict] = []
+    repairs: list[dict] = []
+    for name, c in checks.items():
+        if c.get("ok") is False:
+            blocking.append({"constraint": name, "reason": c.get("status")})
+            if name == "deliverability":
+                repairs.append({"constraint": "deliverability",
+                                "hint": "reduce the cargo, split across a dual-vector strategy, or use an "
+                                        "integrating vehicle"})
+            if name == "legality" and c.get("repair_hint"):
+                repairs.append({"constraint": "legality", "hint": c["repair_hint"]})
+            if name == "reachability":
+                repairs.append({"constraint": "reachability",
+                                "hint": "choose a writable locus (the atlas Site Finder ranks reachable, safe loci)"})
+    feasible = not blocking
+    note = ("all determinable necessary conditions met (feasibility is necessary, not sufficient: efficacy is the "
+            "downstream prediction)" if feasible else "blocked by the named constraint(s); repairs suggested")
+    return SatisfyResult(feasible=feasible, checks=checks, blocking=blocking, repairs=repairs, note=note)

pen_stack/spec/service.py

@@ -0,0 +1,33 @@
+"""WriteSpec service: the one call the surfaces (REST / MCP / web builder) use.
+
+``parse_request`` runs the grounded extractor, plans clarifying questions, and (optionally) the feasibility
+check, returning a single JSON-able payload: the typed WriteSpec, the assumptions behind every inferred field,
+the clarifying questions for anything underspecified, the unresolved terms, the downstream design adapter, and
+the feasibility verdict. Nothing is fabricated: unresolved stays null, inferred is labelled, ambiguous asks.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from pen_stack.spec.clarify import clarifying_questions
+from pen_stack.spec.extract import extract_writespec
+
+
+def parse_request(prose: str, *, overrides: dict | None = None, check_feasibility: bool = True) -> dict[str, Any]:
+    """Parse prose into a typed WriteSpec + clarifications + (optionally) a feasibility verdict."""
+    spec = extract_writespec(prose, overrides=overrides)
+    qs = clarifying_questions(spec)
+    out: dict[str, Any] = {
+        "writespec": spec.model_dump(),
+        "assumptions": spec.assumptions,
+        "clarifications": qs,
+        "actionable": not qs,
+        "unresolved": spec.unresolved,
+        "ontology_validation": spec.ontology_validation(),
+        "legacy_design": spec.to_legacy_design(),
+        "no_fabrication": True,
+    }
+    if check_feasibility:
+        from pen_stack.spec.satisfy import check_satisfiable
+        out["feasibility"] = check_satisfiable(spec).model_dump()
+    return out

pen_stack/spec/writespec.py

@@ -0,0 +1,252 @@
+"""The WriteRequest schema: a typed, ontology-backed genome-writing request.
+
+``WriteRequest`` is an SBOL3 profile expressed as a pydantic model. It carries the write semantics (write-type,
+cargo with Sequence-Ontology roles, target locus/gene/att-site/phenotype, cell type, constraints) PLUS a
+grounding discipline: a per-field provenance map (explicit | inferred | user | unresolved), the assumptions behind
+every inferred field, clarifying questions for underspecified required fields, and the list of terms that could
+not be resolved (kept null, never invented). A WriteRequest is a REQUEST, not a claim.
+
+Round-trips: JSON always; SBOL3 via the real ``sbol3`` library when the ``[spec]`` extra is installed (native
+Components + SO roles for interoperability, with the full typed spec carried losslessly in a PROV-O-namespaced
+annotation); GenBank for a cargo component that carries a DNA sequence. ``to_legacy_design`` is the adapter that
+emits the dict the existing downstream stages (verify / plan / safety) already consume, so the whole stack reads
+one contract without a rewrite.
+"""
+from __future__ import annotations
+
+import re
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+WRITE_TYPES = ["insertion", "excision", "inversion", "replacement", "regulatory_rewrite",
+               "landing_pad_install", "multiplex"]
+
+ProvenanceKind = Literal["explicit", "inferred", "user", "unresolved"]
+
+# id-format validators per ontology (a resolved id must match, else it is flagged not-valid).
+_ID_FORMATS = {
+    "SO": re.compile(r"^SO:\d{7}$"),
+    "Cellosaurus": re.compile(r"^CVCL_[A-Z0-9]{4}$"),
+    "CL": re.compile(r"^CL:\d{7}$"),
+    "MONDO": re.compile(r"^MONDO:\d{7}$"),
+    "HPO": re.compile(r"^HP:\d{7}$"),
+    "ChEBI": re.compile(r"^CHEBI:\d+$"),
+    "HGNC": re.compile(r"^[A-Z0-9-]+$"),
+    "GRCh38": re.compile(r"^chr[0-9XYM]+(:\d+-\d+)?$"),
+}
+
+
+class Resolved(BaseModel):
+    """A free-text term resolved to a canonical ontology id, with confidence and the candidate set when ambiguous."""
+    text: str | None = None              # the surface term from the prose
+    id: str | None = None                # canonical id (CVCL_0063, SO:0000167, MONDO:..., a gene symbol, ...)
+    label: str | None = None             # canonical label
+    ontology: str | None = None          # Cellosaurus | CL | SO | HGNC | MONDO | HPO | ChEBI | GRCh38
+    confidence: float | None = None
+    candidates: list[dict] = Field(default_factory=list)  # ranked alternatives when ambiguous
+    note: str | None = None
+
+    @property
+    def resolved(self) -> bool:
+        return self.id is not None
+
+
+class CargoComponent(BaseModel):
+    """A component to be written (CDS, promoter, insulator, attB, polyA, ...), with its SO/SBO role."""
+    name: str
+    role: Resolved | None = None         # Sequence-Ontology role
+    sequence: str | None = None          # DNA sequence if given (else intent-only)
+    length_bp: int | None = None
+
+
+class Target(BaseModel):
+    """Where the write goes: a locus, a gene, an att/landing site, or a phenotype goal."""
+    kind: Literal["locus", "gene", "att_site", "phenotype", "unspecified"] = "unspecified"
+    gene: Resolved | None = None
+    locus: Resolved | None = None        # GRCh38 coordinate / safe-harbour
+    att_site: str | None = None          # attB/attP/landing-pad id
+    phenotype: Resolved | None = None    # MONDO/HPO disease goal
+    # Further genes named alongside ``gene`` in one request ("knock out TRAC and B2M"). ``gene`` stays the primary
+    # target so every existing consumer reads the same field; the rest are carried here rather than dropped, since
+    # a multiplex request that silently keeps only its first target would be designed against an incomplete spec.
+    additional_genes: list[Resolved] = Field(default_factory=list)
+
+
+class Constraints(BaseModel):
+    """The request's constraints (efficiency floor, scarless, safety switch, copy number, guardrails, delivery)."""
+    efficiency_floor: float | None = None
+    scarless: bool | None = None
+    safety_switch_required: bool | None = None
+    copy_number: int | None = None
+    germline: bool | None = None         # germline guardrail (True = a germline edit was requested -> flagged)
+    max_cargo_bp: int | None = None
+    delivery_limit: str | None = None    # e.g. "non_integrating", "AAV_only"
+    inducer: Resolved | None = None      # ChEBI small-molecule inducer / selection agent
+
+
+class WriteRequest(BaseModel):
+    """A typed, ontology-backed, machine-checkable genome-writing request (an SBOL3 profile). A REQUEST, not a claim.
+
+    Every field that was not explicit in the source prose is recorded in ``provenance`` as ``inferred`` (with the
+    rationale in ``assumptions``); a required field that is unspecified or ambiguous yields a ``clarifications``
+    question rather than a guess; a term that could not be resolved is listed in ``unresolved`` and kept null.
+    """
+    write_type: str                      # one of WRITE_TYPES
+    cargo: list[CargoComponent] = Field(default_factory=list)
+    target: Target = Field(default_factory=Target)
+    cell_type: Resolved | None = None
+    constraints: Constraints = Field(default_factory=Constraints)
+    # --- grounding discipline ---
+    provenance: dict[str, ProvenanceKind] = Field(default_factory=dict)  # field -> origin
+    assumptions: list[str] = Field(default_factory=list)                 # inferred fields + rationale
+    clarifications: list[str] = Field(default_factory=list)              # questions for underspecified fields
+    unresolved: list[str] = Field(default_factory=list)                  # terms that could not be resolved
+    free_text_note: str | None = None    # the original prose (the cargo-function the Guardian must screen)
+    extensions: dict[str, Any] = Field(default_factory=dict)            # edge intents the schema does not cover
+
+    # ---- validation -------------------------------------------------------------------------------------------
+    def ontology_validation(self) -> dict[str, Any]:
+        """Check every resolved field's id against its ontology id-format. Returns the per-field verdict."""
+        bad: list[dict] = []
+        checked = 0
+        for field, r in self._resolved_fields().items():
+            if r.id is None or r.ontology is None:
+                continue
+            checked += 1
+            fmt = _ID_FORMATS.get(r.ontology)
+            if fmt is not None and not fmt.match(r.id):
+                bad.append({"field": field, "id": r.id, "ontology": r.ontology})
+        return {"checked": checked, "invalid": bad, "all_valid": not bad}
+
+    def _resolved_fields(self) -> dict[str, Resolved]:
+        out: dict[str, Resolved] = {}
+        if self.cell_type:
+            out["cell_type"] = self.cell_type
+        if self.target.gene:
+            out["target.gene"] = self.target.gene
+        if self.target.locus:
+            out["target.locus"] = self.target.locus
+        if self.target.phenotype:
+            out["target.phenotype"] = self.target.phenotype
+        if self.constraints.inducer:
+            out["constraints.inducer"] = self.constraints.inducer
+        for i, c in enumerate(self.cargo):
+            if c.role:
+                out[f"cargo[{i}].role"] = c.role
+        return out
+
+    @property
+    def is_grounded(self) -> bool:
+        """No fabrication: every field is either explicit, user, inferred-and-labelled, or unresolved-and-null."""
+        return self.ontology_validation()["all_valid"]
+
+    # ---- the downstream adapter -------------------------------------------------------------------------------
+    def to_legacy_design(self) -> dict:
+        """Emit the dict the existing downstream stages (verify / plan / safety / delivery) already consume.
+
+        This is the wrap-not-rewrite adapter: the whole stack reads ``WriteRequest`` through this one mapping
+        instead of a per-stage rewrite. Unspecified fields fall back to the stack's documented defaults.
+
+        The design it emits carries ONE target, ``target.gene``. A request naming several
+        (``target.additional_genes``) describes several writes, so it needs one design per target: this adapter
+        does not fold them together, and a caller iterating a multiplex request must build a design for each.
+        """
+        cargo_bp = self.constraints.max_cargo_bp
+        if cargo_bp is None:
+            cargo_bp = sum((c.length_bp or 0) for c in self.cargo) or None
+        gene = self.target.gene.id if (self.target.gene and self.target.gene.id) else None
+        locus = self.target.locus.id if (self.target.locus and self.target.locus.id) else None
+        chrom = None
+        if locus and locus.startswith("chr"):
+            chrom = locus.split(":")[0]
+        design = {
+            "write_type": self.write_type,
+            "gene": gene,
+            "chrom": chrom,
+            "cargo_bp": cargo_bp,
+            "cell_type": self.cell_type.id if (self.cell_type and self.cell_type.id) else None,
+            # delivery_limit holds either a named vehicle or the "non_integrating" flag
+            "delivery_vehicle": (self.constraints.delivery_limit
+                                 if self.constraints.delivery_limit not in (None, "non_integrating") else None),
+            "non_integrating": (self.constraints.delivery_limit == "non_integrating") or None,
+            "installed_att": bool(self.target.att_site),
+            "cargo_function": self.free_text_note,
+        }
+        return {k: v for k, v in design.items() if v is not None}
+
+    # ---- serialization ----------------------------------------------------------------------------------------
+    def to_json(self) -> str:
+        return self.model_dump_json(indent=2)
+
+    @classmethod
+    def from_json(cls, s: str) -> "WriteRequest":
+        return cls.model_validate_json(s)
+
+    def to_sbol3(self, namespace: str = "https://penstack.org/writespec") -> str:
+        """Serialize to SBOL3 (sorted N-triples) via the ``sbol3`` library: native Components + Sequence-Ontology
+        roles for interoperability, with the full typed spec carried losslessly in a PROV-O-namespaced annotation.
+
+        Requires the ``[spec]`` extra (``pip install sbol3``); raises ImportError otherwise (never fabricates).
+        """
+        import sbol3
+        sbol3.set_namespace(namespace)
+        doc = sbol3.Document()
+        top = sbol3.Component(f"write_{self.write_type}", sbol3.SBO_DNA)
+        top.name = f"WriteRequest: {self.write_type}"
+        # native SO roles for the cargo (interoperable with the SBOL3 / GenBank ecosystem)
+        for i, c in enumerate(self.cargo):
+            sub = sbol3.Component(f"cargo_{i}_{re.sub(r'[^A-Za-z0-9]', '_', c.name)[:24]}", sbol3.SBO_DNA)
+            if c.role and c.role.id and c.role.ontology == "SO":
+                sub.roles = [f"https://identifiers.org/SO:{c.role.id.split(':')[1]}"]
+            if c.sequence:
+                seq = sbol3.Sequence(f"seq_{i}", elements=c.sequence.upper(), encoding=sbol3.IUPAC_DNA_ENCODING)
+                doc.add(seq)
+                sub.sequences = [seq.identity]
+            doc.add(sub)
+            top.features.append(sbol3.SubComponent(sub))
+        # the full typed spec rides losslessly as a custom (PROV-O-style) annotation
+        top.write_spec_json = sbol3.TextProperty(top, f"{namespace}#writeSpecJson", 0, 1)
+        top.write_spec_json = self.to_json()
+        doc.add(top)
+        return doc.write_string(sbol3.SORTED_NTRIPLES)
+
+    @classmethod
+    def from_sbol3(cls, ttl: str, namespace: str = "https://penstack.org/writespec") -> "WriteRequest":
+        """Reconstruct a WriteRequest from its SBOL3 serialization (the lossless PROV-O annotation)."""
+        import sbol3
+        sbol3.set_namespace(namespace)
+        doc = sbol3.Document()
+        doc.read_string(ttl, sbol3.SORTED_NTRIPLES)
+        for obj in doc.objects:
+            prop = f"{namespace}#writeSpecJson"
+            val = obj._properties.get(prop)
+            if val:
+                raw = str(val[0])
+                return cls.from_json(raw)
+        raise ValueError("no WriteRequest annotation found in the SBOL3 document")
+
+    def to_genbank(self) -> str | None:
+        """Export the cargo as a GenBank record when it carries a DNA sequence; None for an intent-only spec."""
+        seqs = [c for c in self.cargo if c.sequence]
+        if not seqs:
+            return None
+        from io import StringIO
+
+        from Bio.Seq import Seq
+        from Bio.SeqFeature import FeatureLocation, SeqFeature
+        from Bio.SeqRecord import SeqRecord
+        full = "".join(c.sequence.upper() for c in seqs)
+        rec = SeqRecord(Seq(full), id="WRITESPEC", name="WriteSpec", description=f"{self.write_type} cargo")
+        rec.annotations["molecule_type"] = "DNA"
+        pos = 0
+        for c in seqs:
+            ln = len(c.sequence)
+            role = (c.role.label if (c.role and c.role.label) else c.name)
+            rec.features.append(SeqFeature(FeatureLocation(pos, pos + ln), type="misc_feature",
+                                           qualifiers={"label": [role]}))
+            pos += ln
+        buf = StringIO()
+        from Bio import SeqIO
+        SeqIO.write(rec, buf, "genbank")
+        return buf.getvalue()

pen_stack/twin/__init__.py

@@ -0,0 +1,14 @@
+"""pen_stack.twin, the digital twin for genome writing.
+
+A calibrated, scope-bounded write-outcome predictor: mechanism where computable (cassette expression), an
+in-distribution virtual-cell transcriptional-response estimate (OOD-gated), and an immune-outcome dimension
+sourced from the immune profile, fused into one prediction with an interval that widens under OOD and an explicit
+boundary at phenotype. A hypothesis engine, grounded where it is weak; never an oracle of truth.
+"""
+from __future__ import annotations
+
+from pen_stack.twin.calibrate import calibrate_outcome, interval_coverage
+from pen_stack.twin.mechanistic import cassette_expression
+from pen_stack.twin.outcome import predict_outcome
+
+__all__ = ["predict_outcome", "cassette_expression", "calibrate_outcome", "interval_coverage"]

pen_stack/twin/calibrate.py

@@ -0,0 +1,61 @@
+"""Twin calibration, two-sided.
+
+Reports the twin's calibration on held-out (predicted, observed) outcomes WHATEVER the shape: interval coverage
+vs nominal, and a skill comparison against a NAIVE baseline (predict the mean) with a bootstrap CI on the MAE
+gap. A twin "beats" the baseline only when the CI excludes zero in its favour, otherwise the negative is
+reported verbatim. This mirrors the field's own result (Arc VCC): perturbation prediction does not yet
+consistently beat naive baselines, and the twin says so rather than hiding it.
+"""
+from __future__ import annotations
+
+import numpy as np
+
+
+def _mae(pred, obs) -> float:
+    return float(np.mean(np.abs(np.asarray(pred, float) - np.asarray(obs, float))))
+
+
+def _bootstrap_gap(twin_pred, naive_pred, obs, *, reps: int = 300, seed: int = 0) -> tuple[float, list[float]]:
+    """Bootstrap the MAE gap (naive - twin); positive => twin better. Returns mean gap + 95% CI."""
+    rng = np.random.default_rng(seed)
+    twin_pred, naive_pred, obs = map(lambda a: np.asarray(a, float), (twin_pred, naive_pred, obs))
+    n = len(obs)
+    gaps = []
+    for _ in range(reps):
+        idx = rng.integers(0, n, n)
+        gaps.append(_mae(naive_pred[idx], obs[idx]) - _mae(twin_pred[idx], obs[idx]))
+    lo, hi = np.percentile(gaps, [2.5, 97.5])
+    return float(np.mean(gaps)), [round(float(lo), 4), round(float(hi), 4)]
+
+
+def interval_coverage(intervals, obs) -> dict:
+    """Empirical coverage of the prediction intervals vs the observed outcomes."""
+    obs = np.asarray(obs, float)
+    covered = np.array([lo <= y <= hi for (lo, hi), y in zip(intervals, obs)])
+    return {"empirical_coverage": round(float(np.mean(covered)), 4), "n": int(len(obs))}
+
+
+def calibrate_outcome(predictions, observations, *, intervals=None, reps: int = 300, seed: int = 0) -> dict:
+    """Two-sided calibration of the twin against a naive mean baseline. Reports the MAE gap with a
+    bootstrap CI (beats-or-not), and interval coverage when intervals are supplied."""
+    pred = np.asarray(predictions, float)
+    obs = np.asarray(observations, float)
+    n = len(obs)
+    if n < 3:
+        return {"available": False, "n": n, "reason": "too few held-out outcomes to calibrate"}
+    naive = np.full(n, float(np.mean(obs)))
+    mae_twin, mae_naive = _mae(pred, obs), _mae(naive, obs)
+    gap, ci = _bootstrap_gap(pred, naive, obs, reps=reps, seed=seed)
+    beats = bool(ci[0] > 0) # CI excludes 0 in the twin's favour
+    out = {
+        "available": True, "n": n,
+        "mae_twin": round(mae_twin, 4), "mae_naive": round(mae_naive, 4),
+        "mae_gap_naive_minus_twin": round(gap, 4), "gap_ci": ci,
+        "beats_naive_baseline": beats,
+        "honest_note": ("twin beats naive (CI excludes 0)" if beats
+                        else "twin does NOT beat naive on this data (CI spans 0) - reported, not hidden"),
+        "no_fabrication": True,
+    }
+    if intervals is not None:
+        out["interval_coverage"] = interval_coverage(intervals, obs)
+    return out

pen_stack/twin/data/__init__.py

@@ -0,0 +1,12 @@
+"""Position-effect / expression supervision data."""
+from pen_stack.twin.data.position_effect import ( # noqa: F401
+    DATASETS,
+    FEATURE_COLS,
+    SCHEMA,
+    available_datasets,
+    blocked_splits,
+    heldout_celltype_splits,
+    leakage_report,
+    load_position_effect,
+    normalize_within,
+)