noether-physics 0.1.0__py3-none-any.whl

noether/__init__.py

@@ -0,0 +1,11 @@
+"""NOETHER — a verifiable research-paper assistant for physics.
+
+Four guarantees hold across every code path that emits an answer:
+
+1. No unresolvable citation ever leaves the system.
+2. Every claim is anchored to a verbatim span in a real source.
+3. Every equation is parsed, dimensionally checked, and runnable.
+4. Every session is reproducible from a content-hashed record.
+"""
+
+__version__ = "0.1.0"

noether/answer/pipeline.py

@@ -0,0 +1,244 @@
+"""The answering pipeline: gather evidence, judge a draft, record the result.
+
+The division of labour is the point. NOETHER does not write prose and does not
+call a language model. It:
+
+1. **gathers** passages that provably exist in the corpus (``gather_evidence``),
+2. **judges** whatever claims come back against those passages (``submit``),
+3. **records** what survived, what did not, and why.
+
+Claude -- or any caller -- does the writing. That separation is what makes the
+guarantees enforceable: the component that could hallucinate is not the
+component that decides what ships.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from ..compute.symbolic import parse_equation
+from ..compute.units import check_equation
+from ..index.retrieve import Retriever
+from ..index.store import Store
+from ..verify.gate import Claim, Gate
+from .records import (
+    AnswerRecord,
+    EvidenceItem,
+    EvidencePack,
+    RecordedClaim,
+    RecordedEquation,
+    new_answer_id,
+    utc_now,
+)
+
+
+@dataclass
+class DraftClaim:
+    """A claim as submitted by a caller: text plus the evidence it cites."""
+
+    text: str
+    evidence_ids: list[str] = field(default_factory=list)
+
+
+def corpus_hash(store: Store) -> str:
+    """A hash over which papers are in the corpus and their content.
+
+    Part of guarantee 4: an answer is only reproducible against the corpus it
+    was computed from, so the record has to name that corpus exactly.
+    """
+    digest = hashlib.sha256()
+    with store.connect() as conn:
+        rows = conn.execute(
+            "SELECT paper_id, content_hash FROM papers ORDER BY paper_id"
+        ).fetchall()
+    for row in rows:
+        digest.update(row["paper_id"].encode("utf-8"))
+        digest.update(row["content_hash"].encode("utf-8"))
+    return digest.hexdigest()
+
+
+def gather_evidence(
+    store: Store,
+    question: str,
+    limit: int = 12,
+    retriever: Retriever | None = None,
+) -> EvidencePack:
+    """Retrieve passages relevant to a question, ready to be cited.
+
+    Each item gets a short stable id so a caller can reference it precisely
+    rather than by quoting text back at us -- quoting back is exactly where a
+    paraphrase silently becomes a fabricated quote.
+    """
+    retriever = retriever or Retriever(store)
+    hits = retriever.search(question, limit=limit)
+
+    items: list[EvidenceItem] = []
+    papers: dict[str, str] = {}
+    for index, hit in enumerate(hits, start=1):
+        chunk = hit.chunk
+        items.append(
+            EvidenceItem(
+                evidence_id=f"e{index}",
+                paper_id=chunk.paper_id,
+                section_path=chunk.section_path,
+                quote=chunk.text,
+                start=chunk.start,
+                end=chunk.end,
+                kind=chunk.kind,
+                score=round(hit.score, 6),
+            )
+        )
+        if chunk.paper_id not in papers:
+            row = store.paper_row(chunk.paper_id)
+            papers[chunk.paper_id] = row["title"] if row else chunk.paper_id
+
+    return EvidencePack(
+        question=question, items=items, papers=papers, corpus_hash=corpus_hash(store)
+    )
+
+
+def submit(
+    store: Store,
+    pack: EvidencePack,
+    drafts: list[DraftClaim],
+    equations: list[dict[str, Any]] | None = None,
+    references: dict[str, bool] | None = None,
+    answers_dir: Path | None = None,
+) -> AnswerRecord:
+    """Judge a draft against the evidence and produce a record.
+
+    Claims citing evidence that does not exist, or whose quote no longer matches
+    the corpus, are dropped by the gate and listed in ``record.dropped``. A thin
+    answer is visibly thin; it is never padded to look complete.
+    """
+    gate = Gate(store)
+
+    claims: list[Claim] = []
+    unknown_ids: dict[str, str] = {}
+    for draft in drafts:
+        anchors = []
+        for evidence_id in draft.evidence_ids:
+            item = pack.by_id(evidence_id)
+            if item is None:
+                unknown_ids[draft.text] = evidence_id
+                continue
+            anchors.append(item.to_anchor())
+        claims.append(Claim(text=draft.text, anchors=anchors))
+
+    report = gate.check(claims, references=references)
+
+    record = AnswerRecord(
+        answer_id=new_answer_id(pack.question),
+        question=pack.question,
+        claims=[RecordedClaim.from_anchors(c.text, c.anchors) for c in report.accepted],
+        references=dict(references or {}),
+        corpus_hash=pack.corpus_hash,
+        created_at=utc_now(),
+    )
+
+    for rejection in report.rejected:
+        detail = rejection.detail
+        if rejection.claim.text in unknown_ids:
+            detail = f"cited evidence id {unknown_ids[rejection.claim.text]!r}, which was not offered"
+        record.dropped.append(
+            {"claim": rejection.claim.text, "reason": rejection.reason, "detail": detail}
+        )
+    for key, reason in report.dropped_references:
+        record.dropped.append({"claim": "", "reason": "unresolved-reference", "detail": f"{key}: {reason}"})
+
+    for entry in equations or []:
+        record.equations.append(_check_equation_entry(entry))
+
+    if answers_dir is not None:
+        record.save(answers_dir)
+    return record
+
+
+def _check_equation_entry(entry: dict[str, Any]) -> RecordedEquation:
+    """Parse and dimension-check one equation offered for an answer."""
+    latex = entry.get("latex", "")
+    parsed = parse_equation(latex)
+    check = check_equation(parsed, entry.get("units"))
+    return RecordedEquation(
+        latex=latex,
+        paper_id=entry.get("paper_id", ""),
+        number=entry.get("number"),
+        parsed=parsed.ok,
+        dimensions=check.verdict.value,
+        detail=check.detail or parsed.error,
+        notes=list(parsed.notes),
+    )
+
+
+def render_markdown(record: AnswerRecord) -> str:
+    """Render a record for a terminal or a document.
+
+    Anchors are shown inline, because a citation the reader cannot check on the
+    spot is only marginally better than no citation at all.
+    """
+    lines = [f"# {record.question}", ""]
+
+    if not record.claims:
+        lines += [
+            "_No claim in this answer survived verification._",
+            "",
+            "That is a result, not an error: the corpus does not support an answer to",
+            "this question. Ingest more papers, or ask something the corpus covers.",
+            "",
+        ]
+
+    for index, claim in enumerate(record.claims, start=1):
+        lines.append(f"{index}. {claim.text}")
+        for anchor in claim.anchors:
+            quote = anchor["quote"]
+            trimmed = quote if len(quote) <= 160 else quote[:157] + "..."
+            location = anchor["section_path"] or "no section"
+            lines.append(f"   > {trimmed}")
+            lines.append(
+                f"   -- {anchor['paper_id']}, {location}, "
+                f"chars {anchor['start']}-{anchor['end']}"
+            )
+        lines.append("")
+
+    if record.equations:
+        lines += ["## Equations", ""]
+        for equation in record.equations:
+            status = "parsed" if equation.parsed else "did not parse"
+            lines.append(f"- `{equation.latex}`")
+            lines.append(f"  {status}; dimensions: {equation.dimensions}")
+            if equation.detail:
+                lines.append(f"  {equation.detail}")
+            for note in equation.notes:
+                lines.append(f"  note: {note}")
+        lines.append("")
+
+    if record.references:
+        resolved = record.resolved_count
+        lines += ["## References", "", f"{resolved}/{len(record.references)} resolved"]
+        for key, ok in sorted(record.references.items()):
+            lines.append(f"- {'OK ' if ok else 'UNRESOLVED'} {key}")
+        lines.append("")
+
+    if record.dropped:
+        lines += ["## Dropped by the gate", ""]
+        for item in record.dropped:
+            subject = item["claim"][:80] or item["detail"]
+            lines.append(f"- [{item['reason']}] {subject}")
+            if item["claim"] and item["detail"]:
+                lines.append(f"  {item['detail']}")
+        lines.append("")
+
+    # The scope statement sits beside the claims, not buried in the docs:
+    # an anchored claim is one whose evidence exists, not one shown to follow.
+    lines += [
+        "---",
+        "Checked: every quote above is verbatim in the cited paper and every span resolves.",
+        "NOT checked: whether each quote supports the claim drawn from it. A claim can",
+        "contradict its own evidence and still appear here. Read the quotes.",
+        "",
+        f"corpus {record.corpus_hash[:12]} · {record.summary()}",
+    ]
+    return "\n".join(lines)

noether/answer/records.py

@@ -0,0 +1,208 @@
+"""Answer records -- the artifact an answer actually is.
+
+An answer here is not a string. It is a record on disk holding the claims, the
+span each rests on, the references and whether they resolved, the equations with
+their dimensional verdicts, the figures, and the corpus hash the whole thing was
+computed against. That is what makes guarantee 4 -- reproducibility -- more than
+an intention: a record either replays or it does not.
+
+The shape is deliberately serialisable to plain JSON. A record that can only be
+read back by the version of the code that wrote it is not a durable artifact.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from ..ingest.document import Anchor, Span
+
+RECORD_SCHEMA_VERSION = 1
+
+
+@dataclass
+class EvidenceItem:
+    """One retrieved passage offered to whoever is drafting the answer."""
+
+    evidence_id: str
+    paper_id: str
+    section_path: str
+    quote: str
+    start: int
+    end: int
+    kind: str = "paragraph"
+    score: float = 0.0
+
+    def to_anchor(self) -> Anchor:
+        return Anchor(self.paper_id, self.section_path, Span(self.start, self.end), self.quote)
+
+
+@dataclass
+class EvidencePack:
+    """What ``ask_grounded`` hands back: the raw material for an answer.
+
+    Deliberately not a draft. The drafting is done by whatever model is calling
+    us; our job is to supply passages that provably exist and then to judge what
+    comes back against them.
+    """
+
+    question: str
+    items: list[EvidenceItem] = field(default_factory=list)
+    papers: dict[str, str] = field(default_factory=dict)
+    corpus_hash: str = ""
+    #: Instructions returned alongside the evidence, so a caller cannot claim
+    #: not to have known the rules it will be judged by.
+    contract: str = (
+        "Every sentence must cite one or more evidence_id values. A sentence "
+        "without one will be dropped. Do not paraphrase a quote into a stronger "
+        "claim than it supports. If the evidence does not answer the question, "
+        "say so rather than filling the gap."
+    )
+
+    def by_id(self, evidence_id: str) -> EvidenceItem | None:
+        return next((i for i in self.items if i.evidence_id == evidence_id), None)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "question": self.question,
+            "corpus_hash": self.corpus_hash,
+            "contract": self.contract,
+            "papers": self.papers,
+            "evidence": [asdict(i) for i in self.items],
+        }
+
+
+@dataclass
+class RecordedClaim:
+    """A claim that survived the gate, with the spans that carry it."""
+
+    text: str
+    anchors: list[dict[str, Any]] = field(default_factory=list)
+
+    @classmethod
+    def from_anchors(cls, text: str, anchors: list[Anchor]) -> RecordedClaim:
+        return cls(
+            text=text,
+            anchors=[
+                {
+                    "paper_id": a.paper_id,
+                    "section_path": a.section_path,
+                    "start": a.span.start,
+                    "end": a.span.end,
+                    "quote": a.quote,
+                }
+                for a in anchors
+            ],
+        )
+
+
+@dataclass
+class RecordedEquation:
+    """An equation shown in an answer, with its verdicts attached."""
+
+    latex: str
+    paper_id: str = ""
+    number: int | None = None
+    parsed: bool = False
+    dimensions: str = "unknown"
+    detail: str = ""
+    notes: list[str] = field(default_factory=list)
+
+
+@dataclass
+class AnswerRecord:
+    """A complete, replayable answer."""
+
+    answer_id: str
+    question: str
+    claims: list[RecordedClaim] = field(default_factory=list)
+    equations: list[RecordedEquation] = field(default_factory=list)
+    figures: list[dict[str, Any]] = field(default_factory=list)
+    references: dict[str, bool] = field(default_factory=dict)
+    #: What the gate removed, and why. Always present, even when empty: a reader
+    #: needs to see that the check ran and found nothing, not merely find silence.
+    dropped: list[dict[str, str]] = field(default_factory=list)
+    corpus_hash: str = ""
+    created_at: str = ""
+    schema_version: int = RECORD_SCHEMA_VERSION
+
+    @property
+    def resolved_count(self) -> int:
+        return sum(1 for ok in self.references.values() if ok)
+
+    @property
+    def fully_resolved(self) -> bool:
+        """Guarantee 1, restated as a property of the finished record."""
+        return all(self.references.values())
+
+    def summary(self) -> str:
+        parts = [f"{len(self.claims)} claims"]
+        if self.references:
+            parts.append(f"{self.resolved_count}/{len(self.references)} references resolved")
+        if self.equations:
+            checked = sum(1 for e in self.equations if e.dimensions == "consistent")
+            parts.append(f"{checked}/{len(self.equations)} equations dimensionally consistent")
+        if self.dropped:
+            parts.append(f"{len(self.dropped)} dropped")
+        return ", ".join(parts)
+
+    # ---- persistence ---------------------------------------------------
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def save(self, directory: Path) -> Path:
+        directory.mkdir(parents=True, exist_ok=True)
+        path = directory / f"{self.answer_id}.json"
+        path.write_text(
+            json.dumps(self.to_dict(), indent=2, sort_keys=True), encoding="utf-8"
+        )
+        return path
+
+    @classmethod
+    def load(cls, path: Path) -> AnswerRecord:
+        data = json.loads(Path(path).read_text(encoding="utf-8"))
+        version = data.get("schema_version", 0)
+        if version > RECORD_SCHEMA_VERSION:
+            raise ValueError(
+                f"record schema v{version} is newer than this build understands "
+                f"(v{RECORD_SCHEMA_VERSION}); upgrade rather than reading it partially"
+            )
+        return cls(
+            answer_id=data["answer_id"],
+            question=data["question"],
+            claims=[RecordedClaim(**c) for c in data.get("claims", [])],
+            equations=[RecordedEquation(**e) for e in data.get("equations", [])],
+            figures=data.get("figures", []),
+            references=data.get("references", {}),
+            dropped=data.get("dropped", []),
+            corpus_hash=data.get("corpus_hash", ""),
+            created_at=data.get("created_at", ""),
+            schema_version=version,
+        )
+
+    def content_hash(self) -> str:
+        """Hash over the record's substance, excluding the timestamp.
+
+        Two runs of the same question over the same corpus should hash the same,
+        so a changed hash means changed content rather than a changed clock.
+        """
+        payload = self.to_dict()
+        payload.pop("created_at", None)
+        canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def new_answer_id(question: str) -> str:
+    """A short, stable-ish id: time-ordered, with a hash of the question."""
+    stamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S")
+    digest = hashlib.sha256(question.encode("utf-8")).hexdigest()[:6]
+    return f"ans_{stamp}_{digest}"
+
+
+def utc_now() -> str:
+    return datetime.now(UTC).isoformat(timespec="seconds")