rag-wright 0.1.0__py3-none-any.whl

rag_wright/corpus/document_parser.py

@@ -0,0 +1,299 @@
+"""DOCPARSE-1 (ADR-0049 generic-customer lens): the generic raw-document parser -- the SHARED front-end that lets a
+customer's own PDF / DOCX / HTML flow into BOTH ingestion sides.
+
+The docling parse capability already exists (`capabilities/parsing.py`: `DocumentConverter` -> `DoclingDocument`);
+this module adds (a) a BYTES entry point (customer docs arrive as bytes, e.g. from GCS, not a local path) and the
+two PROJECTIONS the two sides need from a parsed document:
+  - `document_to_text`  -> the contract side (`GcsCorpusAdapter.parse_bytes` seam): one text blob per document.
+  - `document_to_sections` -> the compliance side (`RegulationAdapter`'s `[{section, heading, text}]` shape): the
+    document split into sections at its headings, so a policy PDF ingests the same way an eCFR `sections.json` does.
+
+The DoclingDocument API is grounded (framework graph + installed `inspect`): `export_to_markdown()` for text;
+`iterate_items() -> (item, level)` with heading labels SECTION_HEADER / TITLE / FIELD_HEADING (PAGE_HEADER is
+page furniture, not a section boundary)."""
+from __future__ import annotations
+
+import re
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from docling_core.types.doc.labels import DocItemLabel
+
+# Labels that START a new section (a real heading), vs PAGE_HEADER which is running page furniture (ignored).
+_HEADING_LABELS = frozenset({DocItemLabel.SECTION_HEADER, DocItemLabel.TITLE, DocItemLabel.FIELD_HEADING})
+_TEXT_EXTS = frozenset({"txt", "md", "text"})
+
+
+# 0009-WIRE2: a generous per-document OCR ceiling. A degraded multi-page doc escalated whole to the VLM is a
+# batch op (~30s/page), so it needs more than the per-model-call deadline; per-page escalation would let us
+# tighten this. Overridable via `deadline_s`.
+_OCR_PARSE_DEADLINE_S = 600.0
+
+
+def _default_document_parser() -> Any:
+    """0009-WIRE2: the default parser for raw-document bytes is the TIERED OCR parser -- fast OCR, then a
+    scan-quality gate escalates only degraded pages to the VLM (default Gemma-4 via OpenRouter), and flags what
+    even the VLM cannot read. Graceful degrade when no VLM is configured. So BOTH ingestion pipelines and the MCP
+    document tool get degraded-scan handling through this one chokepoint."""
+    from rag_wright.capabilities.parsing import TieredOCRParser
+
+    return TieredOCRParser()
+
+
+def parse_document_bytes(name: str, data: bytes, *, parser: Any = None) -> Any:
+    """Parse raw document BYTES into a `DoclingDocument`. `.txt`/`.md` bytes are wrapped directly; binary docs
+    (PDF/DOCX/HTML) go through docling. `parser` defaults to the tiered OCR parser (0009-WIRE2), injectable for
+    tests. `name` supplies the file extension docling needs to pick a backend."""
+    ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
+    parser = parser or _default_document_parser()
+    # docling reads a file path, not raw bytes -> write to a temp file preserving the extension for backend choice
+    suffix = f".{ext}" if ext else ".txt"
+    tmp = Path(tempfile.mkdtemp(prefix="docparse_")) / f"doc{suffix}"
+    tmp.write_bytes(data)
+    return parser.convert(tmp)
+
+
+async def aparse_document_bytes(name: str, data: bytes, *, parser: Any = None,
+                                deadline_s: float = _OCR_PARSE_DEADLINE_S) -> Any:
+    """ASYNC-bounded document parse (ADR-0057): run the sync `parse_document_bytes` (incl. the tiered VLM
+    escalation -- the slowest call in the pipeline) OFF the event loop via `to_thread`, under a wall-clock
+    `asyncio.timeout` so a hung/slow OCR never stalls the async ingestion. NOTE: `to_thread` cannot cancel the
+    worker thread, so the deadline unblocks the CALLER (raises TimeoutError); the docling parse thread finishes
+    in the background. True cancellation would require routing the vision call through the async model seam."""
+    import asyncio
+
+    async with asyncio.timeout(deadline_s):
+        return await asyncio.to_thread(parse_document_bytes, name, data, parser=parser)
+
+
+def document_to_text(doc: Any) -> str:
+    """A parsed document -> one text blob (docling markdown export) -- the contract side's `parse_bytes` output."""
+    return doc.export_to_markdown()
+
+
+@dataclass(frozen=True)
+class ContentItem:
+    """issue 0014: one chunkable unit of a parsed document's READING-ORDER body. Duck-typed to a docling text
+    item (`.label`, `.level`, `.text`) so the chunker/segmenter consume it unchanged. The reading-order body is a
+    SUPERSET of `document.texts`: a docling TABLE lives in `document.tables` and a figure in `document.pictures`,
+    NEVER in `.texts`, so a `document.texts`-only chunker silently drops them (never chunked, never indexed, never
+    retrievable, and no failure recorded -- the exact 0014 loss). This projection is the single authority for
+    'the document's chunkable content, in reading order'.
+
+    issue 0032: each item also carries its parse-time PAGE provenance (`page`, 1-based, from `prov[0].page_no`)
+    and, when the parser produced one, a `bbox` (l, t, r, b on `page`). This is the provenance the CU-B5 page
+    map threads to a span's citation (a scanned-PDF click-through lands on the right page). `bbox` is best-effort
+    -- omitted where `prov` has none, and dropped when lines are merged into a paragraph (ambiguous then)."""
+
+    label: Any
+    level: Optional[int]
+    text: str
+    page: Optional[int] = None  # issue 0032: 1-based source page (prov[0].page_no); None when prov is absent
+    bbox: Optional[tuple[float, float, float, float]] = None  # (l, t, r, b) on `page`; only when prov has one
+
+
+def _table_content_text(item: Any, doc: Any) -> str:
+    """A TABLE item -> its atomic markdown (issue 0014), caption prefixed when docling captured one. The markdown
+    keeps the header row with the data rows, so the fee/payment schedule retrieves as a unit."""
+    caption = (item.caption_text(doc) or "").strip()
+    body = (item.export_to_markdown(doc) or "").strip()
+    return f"{caption}\n\n{body}".strip() if caption else body
+
+
+def _picture_content_text(item: Any, doc: Any) -> str:
+    """A PICTURE item -> its extractable text (issue 0014): the caption plus any description annotation
+    (a VLM/description the tiered OCR attached). Empty when the figure carries no text -- nothing to index."""
+    parts: list[str] = []
+    caption = (item.caption_text(doc) or "").strip()
+    if caption:
+        parts.append(caption)
+    for annotation in getattr(item, "annotations", None) or []:
+        text = (getattr(annotation, "text", "") or "").strip()  # DescriptionAnnotation (figure description)
+        if text:
+            parts.append(text)
+    return "\n\n".join(parts)
+
+
+_ENDS_SENTENCE = re.compile(r"""[.:;?!]["')\]]*$""")  # a line that completes a sentence (terminal punct + closers)
+
+
+def _ends_sentence(text: str) -> bool:
+    return bool(_ENDS_SENTENCE.search(text.rstrip()))
+
+
+def _starts_new_sentence(text: str) -> bool:
+    """A line that BEGINS a new provision: its first non-space char is a capital, a digit, or an opener
+    (`(`, `[`, quote, `§`, bullet). A lowercase start is a wrapped continuation ('and (ii) ...')."""
+    stripped = text.lstrip()
+    if not stripped:
+        return False
+    c = stripped[0]
+    return c.isupper() or c.isdigit() or c in "([{\"'§•-"
+
+
+def _join_wrapped(prev: str, nxt: str) -> str:
+    """Join a wrapped continuation to its paragraph: de-hyphenate a soft line-break (`Distribu-` + `tor` ->
+    `Distributor`), otherwise a single space."""
+    if prev.endswith("-") and len(prev) >= 2 and prev[-2].isalpha():
+        return prev[:-1] + nxt
+    return f"{prev} {nxt}"
+
+
+def _merge_wrapped_lines(items: list[ContentItem]) -> list[ContentItem]:
+    """DEFRAG-1: reconstruct paragraphs from docling's per-LINE text items. docling emits each PDF text line as its
+    own item; joined with `\\n\\n` and split by `segment_clause`, one clause shatters into per-line fragments that
+    then fail extraction (NEONSYSTEMS: 219 segments / 121 degenerate -> 87 / 9 after this pass). Consecutive
+    `TEXT` items are merged into one paragraph, breaking ONLY when the previous line ends a sentence AND the next
+    starts one (two-sided, so an abbreviation like 'Inc.' followed by a lowercase 'and' does not false-split, and a
+    clean paragraph-per-item document is left untouched). Any NON-text item (heading, table, figure, list item,
+    page furniture) is a hard boundary -- never merged across."""
+    out: list[ContentItem] = []
+    buf: str = ""
+    buf_level: Optional[int] = None
+    buf_page: Optional[int] = None  # issue 0032: the merged paragraph keeps its FIRST line's page (bbox dropped)
+    for item in items:
+        if item.label != DocItemLabel.TEXT:  # heading / table / picture / list-item / furniture -> hard boundary
+            if buf.strip():
+                out.append(ContentItem(label=DocItemLabel.TEXT, level=buf_level, text=buf, page=buf_page))
+            buf, buf_level, buf_page = "", None, None
+            out.append(item)
+            continue
+        line = (item.text or "").strip()
+        if not line:
+            continue
+        if not buf:
+            buf, buf_level, buf_page = line, item.level, item.page
+        elif _ends_sentence(buf) and _starts_new_sentence(line):  # a real paragraph break
+            out.append(ContentItem(label=DocItemLabel.TEXT, level=buf_level, text=buf, page=buf_page))
+            buf, buf_level, buf_page = line, item.level, item.page
+        else:  # a wrapped continuation of the same clause
+            buf = _join_wrapped(buf, line)
+    if buf.strip():
+        out.append(ContentItem(label=DocItemLabel.TEXT, level=buf_level, text=buf, page=buf_page))
+    return out
+
+
+def _prov_page_bbox(node: Any) -> tuple[Optional[int], Optional[tuple[float, float, float, float]]]:
+    """issue 0032: a docling item's page (1-based) and best-effort bbox from `prov[0]` (the same provenance
+    `scan_quality._page_texts` reads). Returns `(None, None)` when the item has no provenance (a test stub or a
+    born-item with none). The bbox is `(l, t, r, b)` when the parser produced one, else `None`."""
+    prov = getattr(node, "prov", None) or []
+    if not prov:
+        return None, None
+    page = getattr(prov[0], "page_no", None)
+    box = getattr(prov[0], "bbox", None)
+    bbox = None
+    if box is not None:
+        try:
+            bbox = (float(box.l), float(box.t), float(box.r), float(box.b))
+        except (AttributeError, TypeError, ValueError):
+            bbox = None
+    return (int(page) if page is not None else None), bbox
+
+
+def _node_text(node: Any) -> str:
+    """issue 0039: a docling ENUMERATED list item (a numbered contract provision) carries its number in `marker`
+    and STRIPS it from `.text` ('1.1.' + text -> text). The chunker/segmenter/provision-detector read the text,
+    so the section number vanishes and `starts_new_provision` never fires -> every provision falls back to the
+    chunk boundary (issue 0038 grouping degrades to chunk-level). Reconstruct the original by prepending the
+    marker (docling's own `orig`), so the number is present exactly where the detector looks. Only for an
+    `enumerated` node with a marker; a bullet/letter marker is prepended too (it restores the original and does
+    NOT trip the numeric section detector, so those list items still fold into their provision)."""
+    text = getattr(node, "text", "") or ""
+    marker = (getattr(node, "marker", "") or "").strip()
+    if marker and getattr(node, "enumerated", False) and not text.lstrip().startswith(marker):
+        return f"{marker} {text}".strip()
+    return text
+
+
+def content_items(doc: Any) -> list[ContentItem]:
+    """A parsed document -> its READING-ORDER chunkable content items (issue 0014). Walks `iterate_items` over the
+    BODY and FURNITURE layers (so everything in `document.texts`, incl. page furniture, is covered), mapping each
+    item to a `ContentItem`: a TABLE -> its atomic markdown, a PICTURE -> caption + description text, any other
+    item -> its `.text`. A table/picture with no extractable text yields an empty-text item (the chunker strips it
+    exactly as it strips an empty text item today) -- present in reading order, never silently missing.
+
+    NO-SILENT-LOSS GUARANTEE (0014, the issue's Q2): any `document.texts` item the reading-order walk did not
+    visit is appended, so the projection is a strict SUPERSET of `.texts` -- a content item can never vanish
+    upstream of the failure accounting the way a table did before this fix."""
+    from docling_core.types.doc.document import ContentLayer
+
+    def _text_item(node: Any) -> ContentItem:
+        page, bbox = _prov_page_bbox(node)
+        return ContentItem(label=getattr(node, "label", None), level=getattr(node, "level", None),
+                           text=_node_text(node), page=page, bbox=bbox)  # issue 0039: keep the enumerated marker
+
+    if not hasattr(doc, "iterate_items"):  # a plain `.texts`-bearing view (a `_SubDocument` slice / test stub):
+        return [_text_item(t) for t in getattr(doc, "texts", None) or []]  # no reading-order body, no tables
+
+    layers = {ContentLayer.BODY, ContentLayer.FURNITURE}
+    items: list[ContentItem] = []
+    seen: set[int] = set()
+    for node, _level in doc.iterate_items(included_content_layers=layers):
+        seen.add(id(node))
+        label = getattr(node, "label", None)
+        if label == DocItemLabel.TABLE and hasattr(node, "export_to_markdown"):
+            text = _table_content_text(node, doc)
+        elif label == DocItemLabel.PICTURE:
+            text = _picture_content_text(node, doc)
+        else:
+            text = _node_text(node)  # issue 0039: an enumerated list item keeps its section-number marker
+        page, bbox = _prov_page_bbox(node)
+        items.append(ContentItem(label=label, level=getattr(node, "level", None), text=text,
+                                 page=page, bbox=bbox))
+    items = _merge_wrapped_lines(items)  # DEFRAG-1: rejoin per-line items into whole-clause paragraphs
+    for text_item in getattr(doc, "texts", None) or []:  # coverage backstop: never drop a `.texts` item
+        if id(text_item) not in seen:
+            items.append(_text_item(text_item))
+    return items
+
+
+def _section_number(heading: str, index: int) -> str:
+    """A short section id: the leading numeric token of the heading (e.g. '1' from '1. Confidentiality'), else the
+    1-based position -- so the compliance citation is stable and human-meaningful."""
+    token = heading.strip().split()[0].rstrip(".").rstrip(")") if heading.strip() else ""
+    return token if token and any(c.isdigit() for c in token) else str(index)
+
+
+def document_to_sections(doc: Any) -> list[dict]:
+    """A parsed document -> `[{section, heading, text, pages, bbox}]` split at its headings -- the compliance side's
+    shape (`RegulationAdapter` ingests exactly this). Body before the first heading is kept as a leading section
+    (heading ""), so nothing is dropped. PAGE_HEADER and whitespace-only items are skipped.
+
+    issue 0043: each section carries `pages` (the source page(s) its items span, from the parse's per-item
+    provenance -- present even on a scan) and a best-effort `bbox`: a single-item section reports that item's box,
+    a multi-item section reports None (a section is not one rectangle, and a fabricated box is worse than none)."""
+    sections: list[dict] = []
+    heading = ""
+    body: list[str] = []
+    pages: set[int] = set()
+    boxes: list[Any] = []  # the (page, bbox) of each item, to pick a single-item section's box
+
+    def _flush() -> None:
+        text = "\n".join(body).strip()
+        if heading or text:  # keep a section if it has a heading OR any body (never emit a fully empty one)
+            bbox = boxes[0] if len(boxes) == 1 else None  # best-effort: only a single-item section has one box
+            sections.append({"section": _section_number(heading, len(sections) + 1), "heading": heading,
+                             "text": text, "pages": sorted(pages), "bbox": bbox})
+
+    for item, _level in doc.iterate_items():
+        label = getattr(item, "label", None)
+        text = (getattr(item, "text", "") or "").strip()
+        page, box = _prov_page_bbox(item)
+        if label in _HEADING_LABELS:
+            _flush()  # close the previous section
+            heading, body, pages, boxes = text, [], set(), []
+            if page is not None:
+                pages.add(page)  # the heading's page belongs to its section
+        elif label == DocItemLabel.PAGE_HEADER:
+            continue  # running page furniture -> neither a boundary nor body
+        elif text:
+            body.append(text)
+            if page is not None:
+                pages.add(page)
+            if box is not None:
+                boxes.append(box)
+    _flush()  # the final section
+    return sections

rag_wright/corpus/edgar.py

@@ -0,0 +1,231 @@
+"""EDGAR entity acquisition logic (T7, docs/archive/plans/Corpus_Acquisition.md).
+
+Pure logic; the throttled/cached fetch and the CLI live in `scripts/acquire_edgar.py`.
+
+`normalize_cik` is the **single** canonical CIK->EntityId normalization point: a CIK is 10-digit
+zero-padded, which is exactly the canonical `EntityId` (T1), so normalization is zero-pad-to-10 then
+validate against the strict contract. T8's registry loader **reuses this exact function** so the two
+cannot drift from the T1 contract.
+
+Name->CIK proposals are mechanical and structurally **UNVERIFIED**: linking a contract party to a
+CIK is itself the entity-resolution problem (FR-C.7), so a fuzzy/mechanical proposal is never ground
+truth. Each proposal carries an explicit `status` (T7 only ever emits UNVERIFIED; VERIFIED is set by
+human verification at T10), and the CLI writes proposals to a separate `proposed/` location. The
+`MatchCoverage` records which parties resolved and which did not, so T10 verification starts from a
+known map.
+"""
+
+from __future__ import annotations
+
+import re
+from enum import Enum
+
+from pydantic import BaseModel
+
+from rag_wright.contracts.identifiers import EntityId
+
+_CIK_PREFIX = re.compile(r"(?i)^cik[-:_ ]*")
+_NON_ALNUM = re.compile(r"[^a-z0-9]+")
+
+# Read this before treating the unresolved set as "not entities" (a note for the T10 verifier).
+# Conservative normalized-conformed-name matching systematically leaves two classes unresolved, by
+# design, not as a bug: (1) private companies and individuals, who are not in EDGAR at all (only
+# public filers are), so a public-company-to-private-counterparty contract resolves one side only;
+# and (2) name variants not close to the conformed name (subsidiaries filing under a parent, former
+# names, DBAs), some of which the submissions former-names data can later help resolve. Both are
+# exactly the cases human verification (T10) exists for. Consequence for the eval: graph entity
+# coverage will be public-filer-centric (FR-C.7), which matters when reading multi-hop results.
+UNRESOLVED_NOTE = (
+    "Unresolved does NOT mean 'not an entity'. Conservative conformed-name matching intentionally "
+    "misses private companies / individuals (absent from EDGAR) and name variants (subsidiaries, "
+    "former names, DBAs). These are the cases human verification (T10) exists for; graph entity "
+    "coverage is therefore public-filer-centric (FR-C.7)."
+)
+
+
+def normalize_cik(raw: int | str) -> EntityId:
+    """Normalize a raw EDGAR CIK (int, unpadded, or ``CIK``-prefixed) to a canonical `EntityId`.
+
+    This is the one place messy EDGAR CIK forms become the canonical identifier; T8 reuses it.
+    """
+    if isinstance(raw, bool):  # bool is an int subclass; reject explicitly
+        raise ValueError("CIK must be an int or digit string, not bool")
+    if isinstance(raw, int):
+        digits = str(raw)
+    elif isinstance(raw, str):
+        digits = _CIK_PREFIX.sub("", raw.strip())
+    else:
+        raise ValueError(f"CIK must be an int or str, got {type(raw).__name__}")
+    if not digits.isdigit():
+        raise ValueError(f"CIK must be numeric, got {raw!r}")
+    if len(digits) > 10:
+        raise ValueError(f"CIK must be at most 10 digits, got {raw!r}")
+    return EntityId.of(digits.zfill(10))
+
+
+def normalize_name(name: str) -> str:
+    """Fold a company/party name to a comparison key (lowercase, alphanumeric runs collapsed)."""
+    return _NON_ALNUM.sub(" ", name.lower()).strip()
+
+
+class MatchStatus(str, Enum):
+    """The verification state of a name->CIK match. T7 emits only UNVERIFIED."""
+
+    UNVERIFIED = "UNVERIFIED"  # mechanical proposal; NOT ground truth until human-verified (T10)
+    VERIFIED = "VERIFIED"  # set only by human verification at T10
+
+
+class NameToCikProposal(BaseModel):
+    """A mechanical, UNVERIFIED name->CIK proposal (never ground truth until verified at T10)."""
+
+    party_name: str
+    proposed_cik: str  # canonical 10-digit EntityId value
+    proposed_conformed_name: str  # the EDGAR conformed name matched
+    match_method: str  # how the proposal was made, e.g. "normalized_conformed_name"
+    status: MatchStatus = MatchStatus.UNVERIFIED
+
+
+class MatchCoverage(BaseModel):
+    """Which parties got a proposed CIK and which did not, so T10 starts from a known map."""
+
+    resolved: list[NameToCikProposal]
+    unresolved: list[str]
+
+
+class LooseCandidate(BaseModel):
+    """A looser token-overlap CIK candidate, shown WITH its evidence for human confirmation.
+
+    A proposal to eyeball, never an auto-commit: the registry name and the tokens it matched on are
+    surfaced so a common-token collision ("Federated", "Premier", "Excite") is caught on the
+    evidence, not confirmed on a bare CIK. Always UNVERIFIED until a human approves it.
+    """
+
+    proposed_cik: str
+    registry_name: str  # the EDGAR conformed name matched (the evidence)
+    matched_tokens: list[str]
+    score: float  # Jaccard token overlap
+    status: MatchStatus = MatchStatus.UNVERIFIED
+
+
+_CIK_TAG = re.compile(r"<cik>(\d+)</cik>", re.IGNORECASE)
+
+
+def parse_browse_edgar_ciks(atom_xml: str) -> list[str]:
+    """CIKs from an EDGAR `browse-edgar ...&output=atom` company-search response (delisted filers
+    included). Returns canonical 10-digit CIKs, deduped in order; multiple means an ambiguous name."""
+    seen: list[str] = []
+    for match in _CIK_TAG.finditer(atom_xml):
+        try:
+            value = normalize_cik(match.group(1)).value
+        except ValueError:
+            continue
+        if value not in seen:
+            seen.append(value)
+    return seen
+
+
+class EdgarEvidence(BaseModel):
+    """Grounded EDGAR evidence for a CIK, to confirm on (former_names resolve dot-com name changes)."""
+
+    proposed_cik: str
+    registry_name: str
+    former_names: list[str] = []
+    tickers: list[str] = []
+    source: str = "edgar_submissions"
+    status: MatchStatus = MatchStatus.UNVERIFIED
+
+
+def parse_submissions_evidence(submissions: dict) -> EdgarEvidence:
+    """The conformed name, former names, and tickers from an EDGAR submissions record."""
+    return EdgarEvidence(
+        proposed_cik=normalize_cik(submissions["cik"]).value,
+        registry_name=submissions.get("name", ""),
+        former_names=[f["name"] for f in submissions.get("formerNames", []) if f.get("name")],
+        tickers=submissions.get("tickers", []),
+    )
+
+
+def former_names(submissions: dict) -> list[str]:
+    """Former company names from an EDGAR submissions record (alias handling; reused by T24)."""
+    return [f["name"] for f in submissions.get("formerNames", []) if f.get("name")]
+
+
+def loose_cik_candidates(
+    name: str, company_tickers: list[dict], *, top_n: int = 3, min_overlap: float = 0.34
+) -> list[LooseCandidate]:
+    """Token-overlap CIK candidates for a mention, ranked, each carrying its match evidence."""
+    query = set(normalize_name(name).split())
+    if not query:
+        return []
+    scored: list[tuple[float, set[str], dict]] = []
+    for row in company_tickers:
+        title_tokens = set(normalize_name(row["title"]).split())
+        overlap = query & title_tokens
+        if not overlap:
+            continue
+        jaccard = len(overlap) / len(query | title_tokens)
+        if jaccard >= min_overlap:
+            scored.append((jaccard, overlap, row))
+    scored.sort(key=lambda s: (-s[0], s[2]["title"]))
+    return [
+        LooseCandidate(
+            proposed_cik=normalize_cik(row["cik_str"]).value,
+            registry_name=row["title"],
+            matched_tokens=sorted(overlap),
+            score=round(jaccard, 3),
+        )
+        for jaccard, overlap, row in scored[:top_n]
+    ]
+
+
+def propose_matches(
+    party_names: list[str], company_tickers: list[dict]
+) -> MatchCoverage:
+    """Propose name->CIK matches mechanically against the EDGAR company registry seed.
+
+    A conservative normalized-conformed-name match: it proposes only where a party's folded name
+    equals an EDGAR conformed name. Everything else is left unresolved for human verification (T10),
+    rather than fuzzy-guessed into a circular answer key.
+    """
+    by_name: dict[str, dict] = {}
+    for row in company_tickers:
+        by_name.setdefault(normalize_name(row["title"]), row)
+
+    resolved: list[NameToCikProposal] = []
+    unresolved: list[str] = []
+    for name in party_names:
+        row = by_name.get(normalize_name(name))
+        if row is None:
+            unresolved.append(name)
+            continue
+        resolved.append(
+            NameToCikProposal(
+                party_name=name,
+                proposed_cik=normalize_cik(row["cik_str"]).value,
+                proposed_conformed_name=row["title"],
+                match_method="normalized_conformed_name",
+            )
+        )
+    return MatchCoverage(resolved=resolved, unresolved=unresolved)
+
+
+def build_edgar_registry(rows, *, aliases_by_cik=None):
+    """ADR-0067: the SEC/EDGAR registry BUILDER (moved off the generic EntityRegistry). Build an EntityRegistry
+    from `company_tickers.json` rows, keyed by a normalized CIK `EntityId`, with `normalize_name` as the surface
+    normalizer. A row whose CIK is invalid is skipped (recorded in `skipped_ids`), never fabricated. The engine's
+    EntityRegistry stays domain-neutral; this SEC builder + normalize_cik live in the SEC layer (the plug-in)."""
+    from rag_wright.ontology.registry import EntityRegistry, RegistryRecord
+
+    aliases_by_cik = aliases_by_cik or {}
+    registry = EntityRegistry(normalize=normalize_name)
+    for row in rows:
+        raw_cik = row["cik_str"]
+        try:
+            entity_id = normalize_cik(raw_cik)
+        except ValueError:
+            registry.skipped_ids.append(str(raw_cik))
+            continue
+        registry.add(RegistryRecord(
+            entity_id=entity_id, canonical_name=row["title"], ticker=row.get("ticker"),
+            aliases=aliases_by_cik.get(entity_id.value, [])))
+    return registry

rag_wright/corpus/gcs_ingestion.py

@@ -0,0 +1,120 @@
+"""PROD-1 (ADR-0049 generic-customer lens): a GCS `CorpusAdapter` — the first REAL source integration.
+
+Where `CuadAdapter` reads a bundled dataset file, this reads a customer's documents straight from a Google Cloud
+Storage prefix (`gs://<bucket>/<prefix>/`), the shape a real onboarding takes: the customer uploads their corpus
+to a bucket, we ingest it. Everything downstream (`run_corpus_ingestion` + the per-document graph) is unchanged
+and corpus-agnostic; only this adapter is GCS-specific.
+
+Parse-by-extension: `.txt`/`.md`/`.text` are read directly (the PROD-1 corpus -- MAUD + ContractNLI -- is text);
+binary customer docs (`.pdf`/`.docx`/`.html`) route through an injected `parse_bytes` seam (docling in production),
+kept injectable so this adapter stays hermetically testable and so the doc-parse dependency is explicit, not
+hidden. The google-cloud-storage client is injectable for the same reason (tests pass a fake; production builds a
+real `storage.Client`).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Iterable, Optional
+
+from rag_wright.subgraphs.contract_ingestion_pipeline import SourceDocument
+
+_TEXT_EXTS = frozenset({"txt", "md", "text"})
+
+
+class GcsCorpusAdapter:
+    """`CorpusAdapter` over `gs://<bucket>/<prefix>`: list blobs under the prefix, read/parse each, yield one
+    `SourceDocument` per document. `client` (a `google.cloud.storage.Client`) and `parse_bytes` (bytes->text for
+    non-text blobs) are injected -- production builds them, tests fake them."""
+
+    def __init__(
+        self,
+        bucket: str,
+        prefix: str,
+        *,
+        limit: int = 0,
+        include: Optional[frozenset] = None,
+        client: Any = None,
+        parse_bytes: Optional[Callable[[str, bytes], str]] = None,
+        parse_doc: Optional[Callable[[str, str, bytes], SourceDocument]] = None,
+    ) -> None:
+        self._bucket = bucket
+        self._prefix = prefix
+        self._limit = limit
+        self._include = include  # if set, only blobs whose BASENAME is in this set (a curated subset ingest)
+        self._client = client
+        self._parse_bytes = parse_bytes
+        # CHUNK-7 (ADR-0058): structure-preserving seam for a non-text blob -- (source_doc_id, name, bytes) ->
+        # a SourceDocument carrying `.parsed` (the real docling parse), so the chunker's structural pass fires.
+        # Preferred over `parse_bytes` (text-only) when both are set; production injects it via the factory.
+        self._parse_doc = parse_doc
+
+    def _get_client(self) -> Any:
+        if self._client is not None:
+            return self._client
+        from google.cloud import storage  # lazy: keep the module import-light + hermetic
+
+        return storage.Client()
+
+    def _to_source(self, blob: Any, source_doc_id: str, meta: dict) -> Any:  # SourceDocument | PendingDocument | None
+        """One blob -> a SourceDocument (or None to skip an empty object). A text blob is read as text; a non-text
+        blob prefers the structure-preserving `parse_doc` seam (carries `.parsed`), else the `parse_bytes` text
+        seam, else raises (no way to ingest a binary document without a parser)."""
+        name = blob.name
+        ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
+        if ext in _TEXT_EXTS:
+            text = blob.download_as_text()
+            return SourceDocument(source_doc_id=source_doc_id, text=text, metadata=meta) if text.strip() else None
+        if self._parse_doc is not None:  # 0009-ASYNC-INGEST: DEFER download+parse (incl. OCR/VLM escalation) so the
+            from rag_wright.subgraphs.contract_ingestion_pipeline import PendingDocument  # ingest parses it
+            _parse = self._parse_doc                                                       # concurrently + bounded
+            return PendingDocument(source_doc_id=source_doc_id,
+                                   parse=lambda: _parse(source_doc_id, name, blob.download_as_bytes()), metadata=meta)
+        if self._parse_bytes is not None:  # legacy text-only seam (no structure)
+            text = self._parse_bytes(name, blob.download_as_bytes())
+            return SourceDocument(source_doc_id=source_doc_id, text=text, metadata=meta) if text.strip() else None
+        raise NotImplementedError(
+            f"non-text blob {name!r}: inject `parse_doc` (structure-preserving, the production default) or "
+            f"`parse_bytes` (text) to ingest PDF/DOCX/HTML customer documents. The PROD-1 corpus is text (.txt).")
+
+    def documents(self) -> Iterable[Any]:  # SourceDocument (text) or PendingDocument (binary, deferred parse)
+        from rag_wright.contracts.identifiers import canonical_source_doc_id
+
+        client = self._get_client()
+        blobs = [b for b in client.list_blobs(self._bucket, prefix=self._prefix) if not b.name.endswith("/")]
+        if self._include is not None:  # curated subset: keep only the named blobs (by basename)
+            blobs = [b for b in blobs if b.name.rsplit("/", 1)[-1] in self._include]
+        if self._limit:
+            blobs = blobs[: self._limit]
+        for blob in blobs:
+            basename = blob.name.rsplit("/", 1)[-1]
+            meta = {"source": "gcs", "bucket": self._bucket, "blob": blob.name}
+            sd = self._to_source(blob, canonical_source_doc_id(basename), meta)
+            if sd is not None:  # None -> empty/whitespace object skipped (a bad upload never dead-letters the run)
+                yield sd
+
+
+def production_gcs_adapter(bucket: str, prefix: str, *, limit: int = 0,
+                          include: Optional[frozenset] = None, cache_dir: str = "data/cache/gcs_parse",
+                          parse_bytes: Optional[Callable[[str, bytes], str]] = None) -> GcsCorpusAdapter:
+    """Build the adapter with a real `google.cloud.storage.Client` (application-default credentials, the same auth
+    gsutil uses). Lazy import so importing this module needs no GCS client. CHUNK-7 (ADR-0058): a non-text
+    customer document (PDF/DOCX/HTML) is docling-parsed with its STRUCTURE PRESERVED (`.parsed`) so the chunker's
+    structural pass fires -- no longer flattened to text. `parse_bytes` is a legacy text-only override."""
+    from google.auth.exceptions import DefaultCredentialsError
+    from google.cloud import storage
+
+    try:
+        client = storage.Client()
+    except DefaultCredentialsError as e:  # actionable message: the python client needs ADC (gsutil uses gcloud auth)
+        raise RuntimeError(
+            "GCS python client needs Application Default Credentials. Run once: "
+            "`gcloud auth application-default login` (or set GOOGLE_APPLICATION_CREDENTIALS to a service-account "
+            "key in production). Note: gsutil/gcloud being authed is NOT sufficient for the python client.") from e
+    parse_doc = None
+    if parse_bytes is None:  # DOCPARSE-1 + CHUNK-7: structure-preserving docling parse for customer documents
+        from rag_wright.subgraphs.contract_ingestion_pipeline import parsed_source_document
+
+        parse_doc = lambda sid, name, data: parsed_source_document(  # noqa: E731
+            sid, name, data, cache_dir=cache_dir)
+    return GcsCorpusAdapter(bucket, prefix, limit=limit, include=include, client=client,
+                            parse_bytes=parse_bytes, parse_doc=parse_doc)