rag-wright 0.1.0__py3-none-any.whl

rag_wright/contracts/value_match.py

@@ -0,0 +1,84 @@
+"""KG-5a: subsumption-aware, canonicalized matching of a query property constraint to a clause's value.
+
+The typed-KG constraint match (Leg B, 4c) was brittle -- exact string set-intersection -- so it lost signal
+to (1) surface variants of the same jurisdiction and (2) values at a finer granularity than the query. This
+module is the fix: `value_satisfies(dimension, query_value, clause_value)` decides whether a clause's stored
+value satisfies a query constraint, using
+
+  - JURISDICTION: canonicalization (England/England and Wales/English law -> england), else normalized string;
+  - SUBSUMPTION rollup: a more specific closed value satisfies a query for its broader value
+    (licensor_affiliates -> affiliates; geographic_and_activity -> geographic AND activity;
+     price_and_terms -> price AND terms);
+  - everything else: exact match.
+
+Additive: no re-extraction, no value-node mutation -- just canonicalization + a small ontology hierarchy
+(also recorded as skos:broader in contract_bridge.ttl). FUTURE (deferred): covered_subject
+{trademark, copyright} -> ip_infringement (legally sound, slightly looser -- add if a query needs it).
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+from rag_wright.contracts.jurisdiction import canonicalize_jurisdiction
+
+
+class NormalizedValue(BaseModel):
+    """CAP-REG-2: the canonical form of a typed value (KG-5a) — a surface value normalized (jurisdiction
+    canonicalization / subsumption) for matching. `changed` is False when the surface was already canonical.
+    The output contract of the `typed_value_normalization` capability."""
+
+    dimension: str
+    surface: str
+    canonical: str
+    changed: bool
+
+# a more-specific closed value -> the broader query value(s) it also satisfies (the three clear cases)
+VALUE_ROLLUP: dict[str, dict[str, set[str]]] = {
+    "covered_parties": {
+        "licensor_affiliates": {"affiliates"},
+        "licensee_affiliates": {"affiliates"},
+    },
+    "restriction_scope": {
+        "geographic_and_activity": {"geographic", "activity"},
+    },
+    "mfn_scope": {
+        "price_and_terms": {"price", "terms"},
+    },
+}
+
+
+def satisfied_values(dimension: str, clause_value: str) -> set[str]:
+    """The query-constraint values a clause value satisfies: itself plus any broader value it rolls up to."""
+    return {clause_value} | VALUE_ROLLUP.get(dimension, {}).get(clause_value, set())
+
+
+def _norm_jurisdiction(v: str) -> str:
+    return canonicalize_jurisdiction(v) or (v or "").strip().lower()
+
+
+def value_satisfies(dimension: str, query_value: str, clause_value: str) -> bool:
+    """Whether a clause's stored `clause_value` satisfies the query's `query_value` on this dimension --
+    jurisdiction-canonicalized, subsumption-aware, exact otherwise."""
+    if dimension == "jurisdiction":
+        return _norm_jurisdiction(query_value) == _norm_jurisdiction(clause_value)
+    return query_value in satisfied_values(dimension, clause_value)
+
+
+def constraint_match_count(query_constraints: set, clause_props: set) -> int:
+    """How many of the query's (dimension, value) constraints the clause's (dimension, value) props satisfy,
+    under canonicalization + subsumption. Replaces the old exact set-intersection count."""
+    return sum(
+        1 for qd, qv in query_constraints
+        if any(cd == qd and value_satisfies(qd, qv, cv) for cd, cv in clause_props)
+    )
+
+
+def register_typed_value_normalization(registry) -> None:
+    """CAP-REG-2: register `typed_value_normalization` (function; KG-5a canonicalization + subsumption)."""
+    registry.register(
+        "typed_value_normalization",
+        contract=NormalizedValue,
+        kind="function",
+        display_name="Typed value normalization",
+    )

rag_wright/corpus/canonicalize.py

@@ -0,0 +1,116 @@
+"""Entity mention canonicalization -- the normalize / reject / cluster method (T23b).
+
+This is the stage the fragmentation exposed as missing, between recognition (T23) and closed-world
+linking (T24): normalize the surface-form variants of one real-world entity to a shared key, reject
+non-entities (template placeholders, role artifacts, bare over-broad tokens), and cluster the
+variants so one entity is one graph node. It is what stops the graph fragmenting (the
+`Bank of America` / `Bank of America, N.A.` / `"<<enter Company Name>>"` problem) and keeps human
+name->CIK verification scaling with entity count, not mention count.
+
+Used first here, under human verification, to canonicalize the T10 relational eval set. These
+normalize/reject/cluster rules harden into the T23b capability -- keep them.
+"""
+
+from __future__ import annotations
+
+import re
+
+from pydantic import BaseModel
+
+# Legal-form suffixes stripped for the clustering key, so "Bank of America" and "Bank of America,
+# N.A." collapse to one key. Applied repeatedly (a name may carry several, e.g. "Co., Ltd.").
+_LEGAL_SUFFIX = re.compile(
+    r"[,\.\s]+\b("
+    r"incorporated|inc|corporation|corp|company|co|llc|l\.?l\.?c|llp|l\.?p|lp|ltd|limited|plc|"
+    r"gmbh|ag|n\.?\s*v\.?|nv|s\.?\s*a\.?|sa|s\.?p\.?a|spa|n\.?\s*a\.?|na|pllc"
+    r")\b\.?$",
+    re.IGNORECASE,
+)
+_NON_ALNUM = re.compile(r"[^a-z0-9]+")
+_APOSTROPHE = re.compile(r"['’‘`]")  # so "Stremick's" == "Stremicks"
+_PLACEHOLDER = re.compile(r"<<|>>|_{2,}|\bxxx+\b|\benter\b|company name|\[\s*\]", re.IGNORECASE)
+# Alias clause markers: everything from the marker on is an alias, not part of the legal name, so
+# "Acme Inc. d/b/a Brand" -> "Acme Inc." and a mention that is only an alias clause ("formerly known
+# as Tradeum, Inc. which d/b/a VerticalNet Solutions") strips to empty and is rejected.
+_ALIAS_MARKER = re.compile(
+    r"\b(formerly known as|doing business as|also known as|"
+    r"f\s*/?\s*k\s*/?\s*a|d\s*/?\s*b\s*/?\s*a|a\s*/?\s*k\s*/?\s*a)\b",
+    re.IGNORECASE,
+)
+# Contract party-definition clause fragments ("...and together with Buyer the Buyer Entities"), not
+# entity names. A mention carrying one of these is a role phrase, not a company.
+_ROLE_PHRASE = re.compile(
+    r"\b(together with|buyer entit|seller entit|the buyer|the seller|buyer the|seller the|"
+    r"the company and|and together|the parties|collectively)\b",
+    re.IGNORECASE,
+)
+
+# Bare over-broad / role words that are not entities on their own (a key equal to one of these,
+# after suffix stripping, is a role label or an over-broad match, not a company).
+_OVERBROAD = frozenset(
+    {
+        "bank", "company", "co", "parties", "party", "buyer", "seller", "purchaser", "vendor",
+        "supplier", "licensor", "licensee", "customer", "client", "contractor", "agent", "lender",
+        "borrower", "guarantor", "corporation", "corp", "trust", "group", "holdings", "holding",
+        "affiliate", "affiliates", "subsidiary", "the company", "the parties",
+        # bare generic-token fragments that are not entities on their own
+        "services", "solutions", "systems", "technologies", "technology", "international",
+        "enterprises", "industries", "communications", "networks", "media", "capital",
+        "management", "ventures", "partners", "associates", "advisors", "advisory", "consulting",
+        "worldwide", "global", "products",
+    }
+)
+
+
+def _strip_alias_clause(name: str) -> str:
+    """Cut an alias clause ('... d/b/a X', '... formerly known as Y'), keeping the name before it."""
+    match = _ALIAS_MARKER.search(name)
+    return name[: match.start()].strip(" ,;(") if match else name
+
+
+def normalize_entity_name(name: str) -> str:
+    """Clustering key: lowercase, surrounding quotes/parens and legal suffixes stripped, whitespace
+    collapsed. Merges `Bank of America`, `Bank of America, N.A.`, `Bank of America, N. A`."""
+    text = _strip_alias_clause(name).strip().strip("\"'()[]").lower()
+    text = _APOSTROPHE.sub("", text)  # possessive: "stremick's" -> "stremicks"
+    prev = None
+    while prev != text:  # strip stacked suffixes ("co., ltd.")
+        prev = text
+        text = _LEGAL_SUFFIX.sub("", text).strip(" ,.")
+    return _NON_ALNUM.sub(" ", text).strip()
+
+
+def is_entity(name: str) -> bool:
+    """Reject non-entities: template placeholders, role artifacts, and bare over-broad tokens."""
+    stripped = name.strip().strip("\"'()[]")
+    if not stripped or _PLACEHOLDER.search(name) or _ROLE_PHRASE.search(name):
+        return False
+    if stripped.lower().startswith("collectively"):
+        return False
+    key = normalize_entity_name(name)
+    return bool(key) and key not in _OVERBROAD
+
+
+class EntityCluster(BaseModel):
+    """One real-world entity: its clustering key, a representative surface form, and all variants."""
+
+    key: str
+    representative: str  # a full surface form (the longest), for display + EDGAR matching
+    variants: list[str]
+
+
+def cluster_entities(names: list[str]) -> list[EntityCluster]:
+    """Cluster surface forms into entities by normalized key (non-entities rejected first)."""
+    groups: dict[str, list[str]] = {}
+    for name in names:
+        if not is_entity(name):
+            continue
+        key = normalize_entity_name(name)
+        groups.setdefault(key, [])
+        if name not in groups[key]:
+            groups[key].append(name)
+    clusters = [
+        EntityCluster(key=key, representative=max(variants, key=len), variants=sorted(variants))
+        for key, variants in groups.items()
+    ]
+    return sorted(clusters, key=lambda c: c.key)

rag_wright/corpus/cuad.py

@@ -0,0 +1,153 @@
+"""CUAD parsing + scanned-PDF detection (T7, docs/archive/plans/Corpus_Acquisition.md).
+
+`party_entities` extracts the real company parties from the CUAD `Parties` annotation, which mixes
+company names with defined-term role labels ("Company", "MA", "Marketing Affiliate"). It keeps only
+names carrying a corporate suffix and drops bare role labels, conservatively: a role label treated
+as an entity would create a *phantom* shared-party link and inflate the multi-hop material with
+false connections. Better to miss a party than to invent a shared one.
+
+`is_scanned_pdf` detects an image-only PDF via the system `pdftotext` using a **low character-count
+threshold, not strict-zero**, so a stray character on a scanned page does not undercount the scanned
+subset (which would starve the vision-to-text path).
+"""
+
+from __future__ import annotations
+
+import ast
+import csv
+import re
+import subprocess
+import tempfile
+from pathlib import Path
+
+from rag_wright.contracts.identifiers import canonical_source_doc_id
+from rag_wright.corpus.selection import ContractMeta
+
+RASTER_DPI = 150  # render resolution for the synthetic image-only PDFs (readable for OCR)
+
+# Corporate-form tokens that mark a party string as a real company (not a role label).
+_CORP_SUFFIX = re.compile(
+    r"\b("
+    r"inc|incorporated|corp|corporation|company|co|llc|l\.?l\.?c|llp|lp|ltd|limited|plc|"
+    r"gmbh|ag|nv|n\.?v|sa|s\.?a|spa|holdings|group|international|technologies|technology|"
+    r"systems|networks|solutions|services|pharmaceuticals|labs|laboratories|ventures|partners|"
+    r"associates|enterprises|industries|bank|trust|capital|media|communications"
+    r")\b",
+    re.IGNORECASE,
+)
+
+# Bare defined-term role labels that must never be treated as entities, even when they coincide
+# with a corporate-form word ("Company", "Co"). A role label as an entity would invent a shared
+# party and inflate the multi-hop material with false links.
+_ROLE_LABELS = {
+    "company", "co", "the company", "parties", "party", "buyer", "seller", "purchaser", "vendor",
+    "supplier", "licensor", "licensee", "customer", "client", "contractor", "agent", "lender",
+    "borrower", "guarantor", "distributor", "reseller", "manufacturer", "consultant", "partner",
+    "affiliate", "marketing affiliate", "ma",
+}
+
+SCANNED_CHAR_THRESHOLD = 300  # first-2-page text below this -> scanned (image-only)
+
+
+def party_entities(raw: str) -> list[str]:
+    """Company parties from a CUAD `Parties` field (a stringified list), role labels dropped."""
+    try:
+        items = ast.literal_eval(raw) if raw and raw.strip().startswith("[") else []
+    except (ValueError, SyntaxError):
+        items = []
+    seen: set[str] = set()
+    out: list[str] = []
+    for item in items:
+        name = str(item).strip()
+        norm = name.lower().rstrip(".").strip()
+        key = name.lower()
+        if name and norm not in _ROLE_LABELS and _CORP_SUFFIX.search(name) and key not in seen:
+            seen.add(key)
+            out.append(name)
+    return out
+
+
+def is_scanned_by_chars(char_count: int, threshold: int = SCANNED_CHAR_THRESHOLD) -> bool:
+    """Threshold decision: fewer than `threshold` extractable characters -> scanned."""
+    return char_count < threshold
+
+
+def pdf_text_chars(pdf_path: Path, *, pages: int = 2) -> int:
+    """Characters `pdftotext` extracts from the first `pages` pages (0 for a purely scanned PDF)."""
+    try:
+        result = subprocess.run(
+            ["pdftotext", "-l", str(pages), "-q", str(pdf_path), "-"],
+            capture_output=True,
+            timeout=60,
+        )
+        return len(result.stdout.decode("utf-8", "ignore").strip())
+    except (subprocess.SubprocessError, OSError):
+        return 0
+
+
+def is_scanned_pdf(pdf_path: Path, *, threshold: int = SCANNED_CHAR_THRESHOLD) -> bool:
+    """Whether a PDF is scanned/image-only (low extractable-text character count)."""
+    return is_scanned_by_chars(pdf_text_chars(pdf_path), threshold)
+
+
+def rasterize_to_image_pdf(src_pdf: Path, dst_pdf: Path, *, dpi: int = RASTER_DPI) -> int:
+    """Render `src_pdf`'s pages to images and write an **image-only** PDF (no text layer) to `dst_pdf`.
+
+    CUAD ships no image-only PDFs (all carry a text layer), so to exercise Docling's OCR /
+    vision-to-text path we deterministically re-render a few contracts as scans: poppler `pdftoppm`
+    rasterizes each page to PNG, Pillow assembles them into a PDF with no extractable text. Given the
+    same source and `dpi` this is reproducible. Returns the page count.
+    """
+    from PIL import Image  # lazy: only the rasterization path needs Pillow
+
+    dst_pdf.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory() as td:
+        prefix = Path(td) / "page"
+        subprocess.run(
+            ["pdftoppm", "-png", "-r", str(dpi), str(src_pdf), str(prefix)],
+            check=True,
+            capture_output=True,
+            timeout=300,
+        )
+        pages = sorted(Path(td).glob("page*.png"))
+        if not pages:
+            raise RuntimeError(f"pdftoppm produced no pages for {src_pdf}")
+        images = [Image.open(p).convert("RGB") for p in pages]
+        images[0].save(dst_pdf, save_all=True, append_images=images[1:])
+        return len(images)
+
+
+def _agreement_type(pdf_path: Path) -> str:
+    """The agreement type is the PDF's immediate folder name, underscores folded to spaces."""
+    return pdf_path.parent.name.replace("_", " ").strip()
+
+
+def load_contract_metadata(extracted_root: Path) -> list[ContractMeta]:
+    """Build `ContractMeta` for every CUAD contract that has both a CSV row and a PDF.
+
+    Runs `pdftotext` per PDF (scanned detection), so this is the slow, I/O part. `extracted_root` is
+    the extracted `CUAD_v1/` directory (holding `master_clauses.csv` and `full_contract_pdf/`).
+    """
+    pdfs = {
+        p.stem.lower(): p
+        for p in (extracted_root / "full_contract_pdf").rglob("*")
+        if p.suffix.lower() == ".pdf"
+    }
+    metas: list[ContractMeta] = []
+    with open(extracted_root / "master_clauses.csv", newline="", encoding="utf-8") as f:
+        for row in csv.DictReader(f):
+            filename = (row.get("Filename") or "").strip()
+            stem = Path(filename).stem.lower()
+            pdf = pdfs.get(stem)
+            if pdf is None:
+                continue
+            metas.append(
+                ContractMeta(
+                    contract_id=canonical_source_doc_id(Path(filename).stem),
+                    agreement_type=_agreement_type(pdf),
+                    parties=party_entities(row.get("Parties", "")),
+                    is_scanned=is_scanned_pdf(pdf),
+                    size_bytes=pdf.stat().st_size,
+                )
+            )
+    return metas

rag_wright/corpus/cuad_ingestion.py

@@ -0,0 +1,72 @@
+"""CUAD ingestion: the reference `CorpusAdapter` + the CUAD ingest driver.
+
+Relocated out of the GENERIC `subgraphs.contract_ingestion_pipeline` (which stays corpus-agnostic) so the
+dataset-specific adapter and driver live with the other per-corpus code in `corpus/`. The generic machinery
+(the `CorpusAdapter` seam, `run_corpus_ingestion`, `production_document_ingest`, and the cache-seed helpers) is
+imported from the pipeline; only the CUAD parsing + wiring lives here. Adding another corpus = a sibling adapter
++ driver here, never touching the generic pipeline.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable
+
+from rag_wright.subgraphs.contract_ingestion_pipeline import (
+    IngestionReport,
+    SourceDocument,
+    aproduction_document_ingest,
+    arun_corpus_ingestion,
+    seed_chunk_cache,
+)
+
+
+class CuadAdapter:
+    """The REFERENCE `CorpusAdapter` (ADR-locked design): CUAD -> `SourceDocument`s. It is the ONLY CUAD-specific
+    code in the ingest path -- it parses the corpus (CUAD ships text in JSON, so no docling parse), assigns the
+    ONE canonical `source_doc_id` (HYG-1), and passes the raw title as metadata. Adding another corpus means
+    writing a sibling adapter (e.g. `AcordAdapter` carrying pre-segmented spans in `metadata`); the pipeline and
+    driver do not change. `run_corpus_ingestion(CuadAdapter(path), ingest_graph, link_fn=...)` ingests it."""
+
+    def __init__(self, cuad_path: Any, *, limit: int = 0) -> None:
+        self._path = cuad_path
+        self._limit = limit
+
+    def documents(self) -> Iterable[SourceDocument]:
+        from rag_wright.contracts.identifiers import canonical_source_doc_id
+        from rag_wright.spans.cuad_labels import parse_cuad
+
+        contracts = list(parse_cuad(self._path))
+        if self._limit:
+            contracts = contracts[: self._limit]
+        for contract in contracts:
+            yield SourceDocument(
+                source_doc_id=canonical_source_doc_id(contract.contract_id),
+                text=contract.context,
+                metadata={"raw_title": contract.contract_id},
+            )
+
+
+async def arun_cuad_ingestion(cuad_path: Any, store: Any, *, cache_dir: Any, limit: int = 0) -> IngestionReport:
+    """INGEST-REFACTOR proof: ingest CUAD through the GENERIC pipeline + `CuadAdapter` -- one call, no
+    `ingest_cuad()`. Builds the EDGAR registry, ensures the schema, ingests `limit` documents. Point `store` at a SCRATCH database for a non-destructive smoke.
+
+    INGEST-REFACTOR (a) cache reuse (the ONLY CUAD-specific wiring): the pipeline reuses GP-1B's party
+    extractions (`dg_extracted_parties.json`, ~482) via `party_seed_path`, and prior `chunk()` manifests
+    (`data/cache/cuad/chunks`, content-hash keyed so only true matches are reused) copied into the run's cache.
+    The unavoidable cost that remains is clause property extraction (the template changed since those were cached)."""
+    import json
+    from pathlib import Path
+
+    from rag_wright.capabilities.dg_extraction import build_verified_registry
+
+    store.ensure_schema()
+    seed_chunk_cache(Path(cache_dir) / "chunks", Path("data/cache/cuad/chunks"))
+    vset = json.loads(Path("data/edgar/verification_set.json").read_text(encoding="utf-8"))
+    ingest_graph = aproduction_document_ingest(
+        store, cache_dir=cache_dir, registry=build_verified_registry(vset),
+        party_seed_path="data/cache/dg_extracted_parties.json")
+    return await arun_corpus_ingestion(
+        CuadAdapter(cuad_path, limit=limit), ingest_graph,
+        # (issue 0028 / ADR-0091: the KG-7 PartyTo link step was retired; no link_fn is wired.)
+        # RESUME-skip: a present Contract node means the whole document already landed (Contract is written last).
+        is_done=lambda doc: store.contract_by_id(doc.source_doc_id) is not None)