pen-stack 0.1.0__py3-none-any.whl

pen_stack/planner/target_site.py

@@ -0,0 +1,118 @@
+"""Target-site / PAM / att-site availability filter.
+
+A **sequence-computable hard filter** on reachability: for a candidate site sequence, does the writer family
+carry the *targeting element* its mechanism physically requires? No usable element → the writer is unreachable
+there and is REJECTED. This extends the WT-KB reachability tier with a sequence-level mechanistic check,
+"the LLM may propose a writer, but if the physics says it cannot engage this site, the funnel rejects it."
+
+Requirement per family (configs/target_sites.yaml):
+  * ``pam``, a PAM motif must occur in the window (Cas9 NGG / Cas12a TTTV / CAST GTN).
+  * ``core_dinuc``, a programmable bipartite recombinase needs its central core dinucleotide (bridge/seek CT).
+  * ``att_site``, a serine integrase needs an attB/attP (pseudo-)att; a naive genomic window has none, so
+                         it REJECTS unless a landing pad was pre-installed (the key mechanistic reject).
+  * ``pe_installable``, a PE-integrase installs its own att, so the site is broadly reachable (available + note).
+
+These are SCREENS, not activity guarantees (relaxed-PAM engineered variants especially); reported with a
+confidence. The point is the hard *negative*: a physically impossible writer, site pairing is rejected.
+"""
+from __future__ import annotations
+
+import re
+from functools import lru_cache
+
+import yaml
+
+from pen_stack._resources import resource
+
+# IUPAC nucleotide → regex character class (for PAM / att-core expansion).
+_IUPAC = {"A": "A", "C": "C", "G": "G", "T": "T", "R": "[AG]", "Y": "[CT]", "S": "[GC]", "W": "[AT]",
+          "K": "[GT]", "M": "[AC]", "B": "[CGT]", "D": "[AGT]", "H": "[ACT]", "V": "[ACG]", "N": "[ACGT]"}
+
+
+@lru_cache(maxsize=1)
+def _cfg() -> dict:
+    return yaml.safe_load(resource("configs/target_sites.yaml").read_text(encoding="utf-8"))
+
+
+def _clean(seq: str) -> str:
+    return re.sub(r"[^ACGT]", "", (seq or "").upper())
+
+
+def iupac_to_regex(motif: str) -> str:
+    return "".join(_IUPAC.get(b, b) for b in motif.upper())
+
+
+def find_motif(seq: str, motif: str) -> list[int]:
+    """0-based start positions of an IUPAC motif (overlapping) in a cleaned sequence."""
+    s = _clean(seq)
+    pat = iupac_to_regex(motif)
+    return [m.start() for m in re.finditer(f"(?=({pat}))", s)]
+
+
+def target_site_available(family: str, seq: str, installed_att: bool = False) -> dict:
+    """Is the writer family's required targeting element present in ``seq``? Returns a structured verdict.
+
+    ``installed_att=True`` declares a pre-installed landing pad (so a serine/PE integrase becomes reachable
+    deterministically). The result is a HARD filter input: ``available=False`` → reject the writer at this site.
+    """
+    fams = _cfg()["families"]
+    fam = fams.get(family)
+    s = _clean(seq)
+    if fam is None:
+        # unknown family: do not silently reject; report not-checkable (reachability falls back to WT-KB tier)
+        return {"family": family, "available": True, "checked": False,
+                "reason": "no target-site rule for this family; reachability deferred to WT-KB tier",
+                "confidence": "none"}
+
+    req = fam["requirement"]
+    conf = fam.get("confidence", "inferred")
+
+    if req == "pam":
+        hits = find_motif(s, fam["pam"])
+        ok = len(hits) > 0
+        out = {"family": family, "requirement": "pam", "pam": fam["pam"], "n_pam_sites": len(hits),
+               "available": ok, "checked": True, "confidence": conf,
+               "reason": (f"{len(hits)} {fam['pam']} PAM site(s) present" if ok
+                          else f"no {fam['pam']} PAM in the window, writer cannot target here (reject)")}
+        if "insertion_offset_bp" in fam:
+            out["insertion_offset_bp"] = fam["insertion_offset_bp"]
+        return out
+
+    if req == "core_dinuc":
+        hits = find_motif(s, fam["core_dinuc"])
+        ok = len(hits) > 0
+        return {"family": family, "requirement": "core_dinuc", "core": fam["core_dinuc"],
+                "n_core_sites": len(hits), "available": ok, "checked": True, "confidence": conf,
+                "reason": (f"central {fam['core_dinuc']} core present ({len(hits)} site(s)); loops reprogrammable"
+                           if ok else f"no {fam['core_dinuc']} core in the window, bipartite recombinase "
+                           "has no recombination point (reject)")}
+
+    if req == "att_site":
+        if installed_att:
+            return {"family": family, "requirement": "att_site", "available": True, "checked": True,
+                    "confidence": conf, "installed_att": True,
+                    "reason": "pre-installed att landing pad declared, integrase reachable deterministically"}
+        present = [m for m in fam.get("att_motifs", []) if iupac_to_regex(m) and re.search(iupac_to_regex(m), s)]
+        ok = bool(present)
+        return {"family": family, "requirement": "att_site", "available": ok, "checked": True,
+                "confidence": conf, "att_motifs_found": present,
+                "reason": ("a native (pseudo-)att core is present" if ok else
+                           "no attB/attP (pseudo-)att in the window, serine integrase needs a pre-installed "
+                           "landing pad; unreachable at a naive site (reject)")}
+
+    if req == "pe_installable":
+        return {"family": family, "requirement": "pe_installable", "available": True, "checked": True,
+                "confidence": conf,
+                "reason": "PE installs the att beacon at the chosen site, broadly reachable (no native motif "
+                          "required; the install step is PE-bounded, per the WT-KB reachability tier)"}
+
+    return {"family": family, "available": True, "checked": False,
+            "reason": f"unhandled requirement {req!r}; reachability deferred to WT-KB tier", "confidence": "none"}
+
+
+def filter_reachable(families: list[str], seq: str, installed_att: bool = False) -> dict:
+    """Split a candidate writer list into mechanistically reachable vs rejected at this site sequence."""
+    verdicts = {f: target_site_available(f, seq, installed_att=installed_att) for f in families}
+    reachable = [f for f, v in verdicts.items() if v["available"]]
+    rejected = [{"family": f, "reason": v["reason"]} for f, v in verdicts.items() if not v["available"]]
+    return {"reachable": reachable, "rejected": rejected, "verdicts": verdicts}

pen_stack/rag/__init__.py

@@ -0,0 +1 @@
+"""pen_stack.rag - see the PEN-STACK program doc."""

pen_stack/rag/corpus.py

@@ -0,0 +1,133 @@
+"""RAG corpus builder.
+
+Assembles the provenance-tagged corpus that grounds the chat's General lane. EVERY chunk is real, already-curated
+repository content - DOI-backed verbatim quotes, the metric guide, the writer-atlas family cards, the data/model
+cards, and the scope boundary - never fabricated paper text. Each chunk carries:
+    chunk_id, text, source_id, doi, access_grade, type, scope_status
+The built table is `data/rag_corpus.parquet` (SHA-locked at build); its embeddings are `data/rag_corpus_emb.npy`.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+
+from pen_stack._resources import project_root
+
+_FIELDS = ["chunk_id", "text", "source_id", "doi", "access_grade", "type", "scope_status"]
+
+
+def _writer_efficiency_chunks() -> list[dict]:
+    from pen_stack.atlas import writer_efficiency as we
+    out = []
+    for r in we.records().itertuples(index=False):
+        q = str(getattr(r, "quote", "") or "").strip()
+        if not q:
+            continue
+        eff = getattr(r, "efficiency_pct", None)
+        eff_s = f" Measured integration efficiency ~{eff}%." if eff is not None and not pd.isna(eff) else ""
+        out.append({
+            "text": (f"{r.system} ({r.family}) at {r.locus} in {r.cell_type}.{eff_s} "
+                     f"Reported: \"{q}\""),
+            "source_id": f"writer_eff:{r.system}:{r.locus}", "doi": str(r.doi),
+            "access_grade": str(getattr(r, "source_access", "unknown")),
+            "type": "efficiency_measurement", "scope_status": "measured"})
+    return out
+
+
+def _metric_guide_chunks() -> list[dict]:
+    from pen_stack.web.guide import metric_guide
+    mg = metric_guide() or {}
+    out = []
+    for key, m in (mg.get("metrics") or {}).items():
+        text = (f"Metric '{m.get('label', key)}' ({key}): scale {m.get('scale')}, {m.get('direction')}. "
+                f"What it means: {m.get('means')} How it is computed: {m.get('computed')} "
+                f"Validation: {m.get('validation')} Reference: {m.get('reference')}")
+        cites = m.get("reference") or ""
+        out.append({"text": text, "source_id": f"metric_guide:{key}", "doi": str(cites),
+                    "access_grade": "curated", "type": "metric_card", "scope_status": "documented"})
+    sd = mg.get("safety_decision")
+    if sd:
+        out.append({"text": "Guardian biosecurity decisions: " + "; ".join(f"{k} = {v}" for k, v in sd.items()),
+                    "source_id": "metric_guide:safety_decision", "doi": "", "access_grade": "curated",
+                    "type": "metric_card", "scope_status": "documented"})
+    return out
+
+
+def _atlas_card_chunks() -> list[dict]:
+    from pen_stack.rag.index import build_cards
+    out = []
+    for c in build_cards():
+        out.append({"text": c.text, "source_id": f"atlas:{c.key}", "doi": ";".join(c.citations),
+                    "access_grade": "curated", "type": "writer_atlas_card", "scope_status": "measured/candidate"})
+    return out
+
+
+def _doc_card_chunks() -> list[dict]:
+    """Chunk the committed data/model cards (docs/cards/*.md) by paragraph."""
+    out = []
+    cards_dir = project_root() / "docs" / "cards"
+    if not cards_dir.exists():
+        return out
+    for p in sorted(cards_dir.glob("*.md")):
+        text = p.read_text(encoding="utf-8", errors="ignore")
+        for para in [s.strip() for s in text.split("\n\n")]:
+            clean = " ".join(para.split())
+            if len(clean) < 80 or clean.startswith("#") and len(clean) < 120:
+                continue
+            out.append({"text": clean[:900], "source_id": f"card:{p.stem}", "doi": "",
+                        "access_grade": "curated", "type": "data_card", "scope_status": "documented"})
+    return out
+
+
+def _scope_chunks() -> list[dict]:
+    """The scope boundary: the known-unknowns the engine never predicts."""
+    from pen_stack.web.guide import pen_stack_facts
+    out = []
+    try:
+        facts = pen_stack_facts() or {}
+    except Exception:  # noqa: BLE001
+        facts = {}
+    ku = facts.get("known_unknowns") or facts.get("scope", {}).get("known_unknowns")
+    if ku:
+        out.append({"text": ("PEN-STACK never predicts these known-unknowns (it abstains): "
+                             + "; ".join(str(x) for x in ku) + ". These are measured clinical endpoints "
+                             "outside the engine's validated envelope."),
+                    "source_id": "scope:known_unknowns", "doi": "", "access_grade": "curated",
+                    "type": "scope_boundary", "scope_status": "known_unknown"})
+    return out
+
+
+def build_corpus() -> pd.DataFrame:
+    """Assemble the full provenance-tagged corpus from real repository content."""
+    rows: list[dict] = []
+    for fn in (_writer_efficiency_chunks, _metric_guide_chunks, _atlas_card_chunks,
+               _doc_card_chunks, _scope_chunks):
+        try:
+            rows.extend(fn())
+        except Exception as e:  # noqa: BLE001 - a missing source must not silently corrupt the corpus; surface it
+            rows.append({"text": f"[source builder {fn.__name__} failed: {e}]", "source_id": "build_error",
+                         "doi": "", "access_grade": "n/a", "type": "build_error", "scope_status": "n/a"})
+    # drop build-error rows from the shippable corpus but keep the count accurate
+    df = pd.DataFrame([r for r in rows if r["type"] != "build_error"])
+    df = df.drop_duplicates(subset=["text"]).reset_index(drop=True)
+    df["chunk_id"] = [f"c{i:04d}" for i in range(len(df))]
+    errors = [r for r in rows if r["type"] == "build_error"]
+    if errors:
+        raise RuntimeError("corpus build had source errors: " + " | ".join(r["text"] for r in errors))
+    return df[_FIELDS]
+
+
+def corpus_path() -> Path:
+    return project_root() / "data" / "rag_corpus.parquet"
+
+
+def emb_path() -> Path:
+    return project_root() / "data" / "rag_corpus_emb.npy"
+
+
+def load_corpus() -> pd.DataFrame:
+    p = corpus_path()
+    if not p.exists():
+        raise FileNotFoundError(f"{p} not built; run scripts/build_rag_corpus.py")
+    return pd.read_parquet(p)

pen_stack/rag/embed.py

@@ -0,0 +1,98 @@
+"""RAG embedding client.
+
+A pinned local embedder (`nomic-embed-text` via the already-running Ollama) for semantic retrieval, with a
+deterministic, model-free **lexical fallback** so the General lane still retrieves (degraded) when no
+embedder is reachable - e.g. in CI or if Ollama is down. The CORPUS embeddings are computed once at build time and
+committed (`data/rag_corpus_emb.npy`); only the live QUERY is embedded at runtime, so retrieval stays reproducible.
+"""
+from __future__ import annotations
+
+import json
+import os
+import urllib.request
+from functools import lru_cache
+
+import numpy as np
+
+EMBED_MODEL = os.getenv("PEN_RAG_EMBED_MODEL", "nomic-embed-text")
+OLLAMA_HOST = os.getenv("OLLAMA_HOST", "http://localhost:11434")
+_TIMEOUT = int(os.getenv("PEN_RAG_EMBED_TIMEOUT", "30"))
+# nomic-embed-text is trained with task prefixes; using them is REQUIRED for retrieval to separate relevant from
+# irrelevant (without them every pair sits in a narrow high-cosine band and abstention cannot discriminate).
+_PREFIX = {"query": "search_query: ", "document": "search_document: "}
+
+
+def embed_text(text: str, task: str = "query") -> np.ndarray | None:
+    """L2-normalised embedding of one string via Ollama (with the nomic task prefix), or None if unreachable."""
+    if os.getenv("PEN_RAG_NO_EMBED") == "1":
+        return None
+    try:
+        req = urllib.request.Request(
+            f"{OLLAMA_HOST}/api/embeddings",
+            data=json.dumps({"model": EMBED_MODEL, "prompt": _PREFIX.get(task, "") + (text or "")}).encode(),
+            headers={"Content-Type": "application/json"})
+        with urllib.request.urlopen(req, timeout=_TIMEOUT) as r:
+            vec = json.loads(r.read().decode())["embedding"]
+        v = np.asarray(vec, dtype="float32")
+        n = float(np.linalg.norm(v))
+        return v / n if n else v
+    except Exception:  # noqa: BLE001 - any failure -> caller uses the lexical fallback
+        return None
+
+
+# latency fix: cache live query embeddings in-process. The chat embeds the user's message on every General-lane
+# request; repeat phrasings ("hi", "what is DNA", "what is gene editing") would otherwise hit Ollama every time. The
+# document side is unaffected - corpus embeddings are computed once at build time and committed.
+@lru_cache(maxsize=256)
+def _embed_query_cached(text: str) -> tuple | None:
+    v = embed_text(text, task="query")
+    return tuple(float(x) for x in v) if v is not None else None
+
+
+def embed_query(text: str) -> np.ndarray | None:
+    """Cached query-side embedding. Identical to `embed_text(text, task='query')` but memoised by text."""
+    t = (text or "").strip()
+    if not t:
+        return embed_text(t, task="query")
+    cached = _embed_query_cached(t)
+    return np.asarray(cached, dtype="float32") if cached is not None else None
+
+
+def embed_corpus(texts: list[str]) -> np.ndarray:
+    """Embed the corpus DOCUMENTS at BUILD time. Raises if the embedder is unreachable (the build must be
+    deterministic and must never silently fall back to a different representation)."""
+    out = []
+    for t in texts:
+        v = embed_text(t, task="document")
+        if v is None:
+            raise RuntimeError(f"embedder '{EMBED_MODEL}' unreachable at {OLLAMA_HOST}; cannot build corpus embeddings")
+        out.append(v)
+    return np.vstack(out).astype("float32")
+
+
+# --- deterministic, model-free lexical fallback (content-token Jaccard) --------------------------------------
+# Stopwords are dropped so the fallback discriminates on CONTENT words: an out-of-corpus query (e.g. "what is the
+# capital of France") must not match a genome-writing chunk via shared function words and slip past the abstention.
+_STOP = frozenset((
+    "the and for are was were that this with from have has had not but you your his her its our their they them "
+    "what which who whom whose when where why how does did doing done can could should would will shall may might "
+    "into onto over under above below than then them they about after before between because while during each "
+    "any all some more most much many few any both either neither nor yet via per such only also same other "
+    "one two three get got make made use used using like just very too here there out off own").split())
+
+
+def tokenize(s: str) -> set[str]:
+    return {w for w in "".join(c.lower() if c.isalnum() else " " for c in (s or "")).split()
+            if len(w) > 2 and w not in _STOP}
+
+
+def lexical_scores(query: str, texts: list[str]) -> np.ndarray:
+    """Jaccard overlap of the query tokens with each document's tokens. Deterministic; no model."""
+    q = tokenize(query)
+    if not q:
+        return np.zeros(len(texts), dtype="float32")
+    out = np.zeros(len(texts), dtype="float32")
+    for i, t in enumerate(texts):
+        d = tokenize(t)
+        out[i] = len(q & d) / len(q | d) if d else 0.0
+    return out

pen_stack/rag/ground.py

@@ -0,0 +1,131 @@
+"""RAG grounding for the chat's General lane.
+
+Retrieval is ADDITIVE, not a gate. The General lane ANSWERS general and social questions by default (restoring the
+hybrid answering behaviour), clearly LABELLED as general knowledge - a labelled general answer is not a
+fabrication, because it is never presented as a PEN-STACK result. Corpus retrieval, when it finds a relevant chunk,
+UPGRADES the answer to 'literature-cited' (with its sources). Abstention is the rare exception: it fires only for a
+SPECIFIC, unsourceable empirical / quantitative genome-writing claim the corpus does not cover and the engine is
+not computing - the genuine citation-or-silence case - so the model cannot dress a fabricated statistic as fact.
+
+Branches:
+  social    -> a friendly answer + a pointer to what the engine can compute (no retrieval, no abstention).
+  cited     -> a relevant corpus chunk was retrieved -> cite-or-silence answer, provenance 'literature-cited'.
+  general   -> background / textbook knowledge -> answer from the LLM, labelled (provenance 'general').
+  abstained -> a specific unsourceable empirical claim -> decline + redirect to the engine (provenance 'abstained').
+"""
+from __future__ import annotations
+
+import os
+import re
+
+from pen_stack.rag.embed import tokenize
+from pen_stack.rag.retrieve import retrieve
+
+# nomic-embed-text compresses cosines into a narrow high band, so an absolute cosine alone cannot separate a valid
+# in-corpus query from an off-topic one. The hybrid gate (cosine floor + lexical content-overlap, with a high-cosine
+# escape) decides only whether the corpus is a relevant SOURCE to cite - NOT whether to answer at all.
+THRESHOLD = float(os.getenv("PEN_RAG_THRESHOLD", "0.50"))
+_SEM_HIGH = float(os.getenv("PEN_RAG_THRESHOLD_HIGH", "0.72"))
+_LEX_THRESHOLD = float(os.getenv("PEN_RAG_LEX_THRESHOLD", "0.05"))
+
+# A social / conversational opener: answered naturally, never gated on the corpus.
+_SOCIAL = re.compile(r"^\s*(hi|hii+|hey+|hello+|yo|hola|sup|greetings|good (morning|afternoon|evening|day)|"
+                     r"thanks?|thank you|cheers|ok(ay)?|cool|nice|great|awesome|who are you|what are you|"
+                     r"how are you|what'?s up|introduce yourself|help)\b", re.I)
+
+# A SPECIFIC genome-writing QUANTITATIVE / empirical claim (not definitional / mechanistic background). When the
+# corpus has no source for it and it is in the general lane (so the engine is not computing it), we abstain +
+# redirect rather than let the model invent a statistic dressed up as fact.
+_SPECIFIC_EMPIRICAL = re.compile(
+    r"\b(efficienc\w*|integration rate|specificit\w*|on-?target rate|off-?target rate|knock-?in rate|titer|titre|"
+    r"affinit\w*|\bkd\b|ic50|fold[- ]improvement|expression level|copy number|"
+    r"percent\w* (of )?(integration|editing|knock|insertion))\b", re.I)
+
+_SOCIAL_REPLY = ("Hi - I'm PEN-STACK's genome-writing co-scientist. Ask a genome-writing question (where to insert, "
+                 "which writer, delivery, immune risk, off-targets, safety) and the engine computes a grounded, "
+                 "traceable answer. I can also answer general background questions, clearly labelled as such.")
+_GENERAL_NO_LLM = ("I can answer general background questions when the language model is enabled. Meanwhile, "
+                   "PEN-STACK's engine can compute grounded, specific answers for your case.")
+_LABEL = "Literature-cited (corpus) - not a PEN-STACK-computed result:"
+_ABSTAIN_SPECIFIC = ("I don't have a grounded source for that specific value and I won't invent one. PEN-STACK's "
+                     "engine can compute it from a concrete design - give me the gene, cell type, and cargo (or "
+                     "the writer/guide) and I'll run the site, writer-efficiency, or off-target tools.")
+
+
+def _is_grounded(query: str, hits: list[dict], method: str) -> bool:
+    """Is the corpus a relevant SOURCE to cite for this query?"""
+    if not hits:
+        return False
+    top = hits[0]
+    if not method.startswith("semantic"):
+        return top["score"] >= _LEX_THRESHOLD
+    lexical_overlap = bool(tokenize(query) & tokenize(top["text"]))
+    return top["score"] >= THRESHOLD and (lexical_overlap or top["score"] >= _SEM_HIGH)
+
+
+def _dedup_sources(hits: list[dict]) -> list[dict]:
+    seen, out = set(), []
+    for h in hits:
+        sid = h["source_id"]
+        if sid in seen:
+            continue
+        seen.add(sid)
+        out.append({"source_id": sid, "doi": h.get("doi") or None, "type": h["type"], "score": round(h["score"], 3)})
+    return out
+
+
+def _cited_answer(message: str, r: dict, allow_llm: bool) -> dict:
+    floor = THRESHOLD if r["method"].startswith("semantic") else _LEX_THRESHOLD
+    grounded_hits = [h for h in r["hits"] if h["score"] >= floor] or r["hits"][:2]
+    sources = _dedup_sources(grounded_hits)
+    deterministic = _LABEL + "\n\n" + "\n\n".join(
+        f"- {h['text']}  [{h['source_id']}{(' · ' + h['doi']) if h.get('doi') else ''}]" for h in grounded_hits)
+    base = {"status": "grounded", "sources": sources, "provenance": "literature-cited", "grounded": True,
+            "retrieval": r["method"], "top_score": r["top_score"]}
+    if allow_llm:
+        from pen_stack.web.llm import _enforce_grounding, _run_llm
+        from pen_stack.web.tools import extract_grounded_numbers
+        ctx = "\n\n".join(f"[{i + 1}] ({h['source_id']}{(' DOI ' + h['doi']) if h.get('doi') else ''}) {h['text']}"
+                          for i, h in enumerate(grounded_hits))
+        system = ("You answer ONLY from the SOURCES provided. After each claim, cite the bracketed [n] it came from. "
+                  "If the sources do not cover the question, say so. Never add a number, name, or vehicle that is "
+                  "not in the sources. Never present anything as a PEN-STACK-computed result.")
+        text, backend = _run_llm(f"SOURCES:\n{ctx}\n\nQUESTION: {message}\n\nAnswer concisely, citing [n].", system,
+                                 kind="general")  # fast-path: short timeout + token cap for cited answers
+        if text:
+            allow = extract_grounded_numbers({"hits": [h["text"] for h in grounded_hits]})
+            return {**base, "reply": _LABEL + "\n\n" + _enforce_grounding(text.strip(), allow), "backend": backend}
+    return {**base, "reply": deterministic, "backend": "deterministic"}
+
+
+def ground_general(message: str, *, allow_llm: bool = True) -> dict:
+    """Return the chat envelope for the General lane: answer (social / general / cited) by default; abstain only on a
+    specific unsourceable empirical claim."""
+    msg = (message or "").strip()
+    if _SOCIAL.match(msg):
+        return {"status": "social", "reply": _SOCIAL_REPLY, "sources": [], "provenance": "general",
+                "grounded": False, "backend": "deterministic", "retrieval": "none", "top_score": 0.0}
+
+    r = retrieve(msg, k=4)
+    if _is_grounded(msg, r["hits"], r["method"]):
+        return _cited_answer(msg, r, allow_llm)
+
+    # no corpus source. A specific empirical genome-writing claim -> abstain + redirect to the engine.
+    if _SPECIFIC_EMPIRICAL.search(msg):
+        return {"status": "abstained", "reply": _ABSTAIN_SPECIFIC, "sources": [], "provenance": "abstained",
+                "grounded": False, "backend": "deterministic", "retrieval": r["method"], "top_score": r["top_score"]}
+
+    # general / background knowledge -> ANSWER, clearly labelled as general (never a PEN-STACK result).
+    if allow_llm:
+        from pen_stack.web.llm import SYSTEM_GENERAL, _GENERAL_LABEL, _run_llm
+        text, backend = _run_llm(
+            f"USER: {msg}\n\nAnswer from general knowledge, clearly and at a graduate level. Do not present anything "
+            f"as a PEN-STACK result.", SYSTEM_GENERAL, kind="general")  # fast-path timeout
+        if text:
+            return {"status": "general", "reply": _GENERAL_LABEL + "\n\n" + text.strip(), "sources": [],
+                    "provenance": "general", "grounded": False, "backend": backend, "retrieval": r["method"],
+                    "top_score": r["top_score"]}
+    from pen_stack.web.llm import _GENERAL_LABEL
+    return {"status": "general", "reply": _GENERAL_LABEL + "\n\n" + _GENERAL_NO_LLM, "sources": [],
+            "provenance": "general", "grounded": False, "backend": "deterministic", "retrieval": r["method"],
+            "top_score": r["top_score"]}

pen_stack/rag/index.py

@@ -0,0 +1,53 @@
+"""Grounded document index for the PEN-STACK RAG.
+
+Builds a cited corpus of fact cards from the curated atlas + WT-KB (each card carries its source DOIs),
+so retrieval-grounded answers always have a citation. If PaperQA + an LLM are available they can index a
+literature corpus on top; the keyword retriever here is the dependency-light default that guarantees the
+"every factual claim is cited" contract without any model.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import pandas as pd
+
+_ATLAS = Path(__file__).resolve().parents[1] / "atlas" / "atlas.parquet"
+
+
+@dataclass
+class Card:
+    key: str
+    text: str
+    citations: list[str] = field(default_factory=list)
+
+
+def build_cards(atlas_parquet: str | Path = _ATLAS) -> list[Card]:
+    """One fact card per writer family, summarising its measured targeting + readiness, with DOIs."""
+    df = pd.read_parquet(atlas_parquet)
+    cards: list[Card] = []
+    for fam, sub in df.groupby("family"):
+        core = sub[sub["entry_kind"].isin(["curated_core", "curated_rep"])]
+        rep = core.iloc[0] if len(core) else sub.iloc[0]
+        dois: list[str] = []
+        for d in core["key_dois"] if len(core) else sub["key_dois"]:
+            dois.extend(str(x) for x in list(d) if str(x).strip())
+        text = (f"Writer family {fam}: representative {rep['representative_system']}; "
+                f"mechanism {rep.get('mechanism_bucket')}; targeting {rep.get('targeting_modality')}; "
+                f"reachability {rep.get('reachability_tier')}; deliverability {rep.get('deliv_class')}; "
+                f"cargo {rep.get('cargo_capacity_bp')} bp; human-cell activity: "
+                f"{rep.get('human_cell_activity')}. {len(sub):,} systems catalogued.")
+        cards.append(Card(key=fam, text=text, citations=sorted(set(dois))))
+    return cards
+
+
+def retrieve(question: str, cards: list[Card], k: int = 3) -> list[Card]:
+    """Keyword overlap retriever (lower-cased token Jaccard). Deterministic, no model needed."""
+    q = set(_tok(question))
+    scored = [(len(q & set(_tok(c.text + " " + c.key))), c) for c in cards]
+    scored.sort(key=lambda x: x[0], reverse=True)
+    return [c for n, c in scored if n > 0][:k]
+
+
+def _tok(s: str) -> list[str]:
+    return [w for w in "".join(ch.lower() if ch.isalnum() else " " for ch in s).split() if len(w) > 2]