rag-wright 0.1.0__py3-none-any.whl

rag_wright/okf/compile.py

@@ -0,0 +1,292 @@
+"""OKF bundle compile (FR-K.1/K.4, T46): deterministic bundle write from sidecar text + enrichment.
+
+Given the chunk texts (from the T40 sidecar) and the per-clause enrichment (category + description), write
+an OKF v0.1 conformant bundle: one markdown file per clause (non-empty `type` frontmatter, body byte-faithful
+to the sidecar text), organized `root -> <category>/ -> <clause>.md`, with an `index.md` per directory and the
+bundle root stamped with `okf_version` and the compile-recipe version. No re-chunk; `chunk_id` is unchanged.
+
+Deterministic (no model call) and content-hash gated: recompiling an unchanged corpus under the same recipe
+does effectively no work. `recompile_category` rewrites one subtree (T51's fast-iteration path).
+
+Registered under the canonical slug `okf_compile` (category 3, a foundation derivation: internal registry
+entry, no ARD manifest -- same shape as `ontology_registry_derivation`).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from rag_wright.capabilities.registry import CapabilityRegistry
+from rag_wright.okf.document import parse_okf, serialize_okf
+from rag_wright.okf.enrich import EnrichedClause
+
+OKF_VERSION = "0.1"
+RECIPE_VERSION = "okf-acord-v1"
+CLAUSE_TYPE = "Clause"
+UNCATEGORIZED = "_uncategorized"
+_INDEX = "index.md"
+_MANIFEST = ".okf_manifest.json"
+
+
+class OkfBundleManifest(BaseModel):
+    """The compile result and the `okf_compile` capability contract: what the bundle root records."""
+
+    okf_version: str
+    recipe_version: str
+    n_clauses: int
+    n_categorized: int
+    categories: dict[str, int]  # category label -> clause-file count (UNCATEGORIZED for abstentions)
+    content_hash: str
+
+
+def _slug(label: str) -> str:
+    """Filesystem-safe directory slug for a category label (`IP Ownership/License` -> `ip-ownership-license`)."""
+    s = re.sub(r"[^a-z0-9]+", "-", label.lower()).strip("-")
+    return s or UNCATEGORIZED
+
+
+def _source_doc_id(chunk_id: str) -> str:
+    return chunk_id.rsplit(":", 2)[0]  # <source_doc_id>:<chunk_index>:<content_hash>
+
+
+def _dir_for(e: EnrichedClause) -> str:
+    return _slug(e.category) if e.categorized else UNCATEGORIZED
+
+
+def _concept_document(chunk_id: str, text: str, e: EnrichedClause) -> str:
+    frontmatter = {
+        "type": CLAUSE_TYPE,
+        "title": _source_doc_id(chunk_id),
+        "description": e.description,
+        "tags": [e.category] if e.categorized else [],
+        "category": e.category if e.categorized else UNCATEGORIZED,
+        "chunk_id": chunk_id,
+        "source_doc_id": _source_doc_id(chunk_id),
+    }
+    return serialize_okf(frontmatter, text)
+
+
+def _bundle_hash(
+    texts: dict[str, str], enrichment: dict[str, EnrichedClause], recipe_version: str
+) -> str:
+    """A content hash over (chunk_id, category, description) plus the recipe version. chunk_id already
+    embeds the text content hash, so an unchanged clause + unchanged enrichment + unchanged recipe hashes
+    the same, and the compile gate skips."""
+    h = hashlib.sha256()
+    h.update(recipe_version.encode("utf-8"))
+    for cid in sorted(texts):
+        e = enrichment[cid]
+        h.update(f"\x00{cid}\x00{e.categorized}\x00{e.category}\x00{e.description}".encode("utf-8"))
+    return h.hexdigest()
+
+
+def _index_text(entries: list[tuple[str, str, str]], *, heading: str) -> str:
+    """Render an index.md section: `* [title](link) - description`, sorted by title."""
+    lines = [f"# {heading}", ""]
+    for title, link, desc in sorted(entries, key=lambda e: e[0].lower()):
+        suffix = f" - {desc}" if desc else ""
+        lines.append(f"* [{title}]({link}){suffix}")
+    return "\n".join(lines) + "\n"
+
+
+def _write_category_index(category_dir: Path) -> None:
+    entries: list[tuple[str, str, str]] = []
+    for md in sorted(category_dir.glob("*.md")):
+        if md.name == _INDEX:
+            continue
+        fm, _ = parse_okf(md.read_text(encoding="utf-8"))
+        entries.append((str(fm.get("title") or md.stem), md.name, str(fm.get("description") or "")))
+    if entries:
+        (category_dir / _INDEX).write_text(_index_text(entries, heading="Clauses"), encoding="utf-8")
+
+
+def _write_root_index(root: Path, *, okf_version: str, recipe_version: str) -> None:
+    entries: list[tuple[str, str, str]] = []
+    for sub in sorted(p for p in root.iterdir() if p.is_dir()):
+        n = len([m for m in sub.glob("*.md") if m.name != _INDEX])
+        entries.append((sub.name, f"{sub.name}/{_INDEX}", f"{n} clauses"))
+    frontmatter = {"okf_version": okf_version, "compile_recipe_version": recipe_version}
+    body = _index_text(entries, heading="Categories")
+    (root / _INDEX).write_text(serialize_okf(frontmatter, body), encoding="utf-8")
+
+
+def compile_bundle(
+    texts: dict[str, str],
+    enrichment: dict[str, EnrichedClause],
+    out_root: Path,
+    *,
+    recipe_version: str = RECIPE_VERSION,
+    okf_version: str = OKF_VERSION,
+) -> OkfBundleManifest:
+    """Compile the whole bundle. Content-hash gated: an unchanged corpus+recipe returns without rewriting."""
+    out_root = Path(out_root)
+    content_hash = _bundle_hash(texts, enrichment, recipe_version)
+    manifest_path = out_root / _MANIFEST
+    if manifest_path.exists():
+        prior = OkfBundleManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
+        if prior.content_hash == content_hash and prior.recipe_version == recipe_version:
+            return prior  # unchanged corpus + recipe -> effectively no work
+
+    out_root.mkdir(parents=True, exist_ok=True)
+    categories: dict[str, int] = {}
+    touched_dirs: set[Path] = set()
+    for chunk_id, text in sorted(texts.items()):
+        e = enrichment[chunk_id]
+        category_dir = out_root / _dir_for(e)
+        category_dir.mkdir(parents=True, exist_ok=True)
+        (category_dir / f"{_source_doc_id(chunk_id)}.md").write_text(
+            _concept_document(chunk_id, text, e), encoding="utf-8"
+        )
+        label = e.category if e.categorized else UNCATEGORIZED
+        categories[label] = categories.get(label, 0) + 1
+        touched_dirs.add(category_dir)
+
+    for category_dir in sorted(touched_dirs):
+        _write_category_index(category_dir)
+    _write_root_index(out_root, okf_version=okf_version, recipe_version=recipe_version)
+
+    manifest = OkfBundleManifest(
+        okf_version=okf_version,
+        recipe_version=recipe_version,
+        n_clauses=len(texts),
+        n_categorized=sum(1 for e in enrichment.values() if e.categorized),
+        categories=dict(sorted(categories.items())),
+        content_hash=content_hash,
+    )
+    manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
+    return manifest
+
+
+def recompile_category(
+    out_root: Path,
+    category_label: str,
+    texts: dict[str, str],
+    enrichment: dict[str, EnrichedClause],
+    *,
+    recipe_version: str = RECIPE_VERSION,
+    okf_version: str = OKF_VERSION,
+) -> None:
+    """Rewrite a single category subtree (its clause files + index) and refresh the root index only.
+
+    T51's fast-iteration path: a signpost-recipe change for one category costs one subtree, not a full
+    rebuild. `texts`/`enrichment` hold just that category's clauses.
+    """
+    out_root = Path(out_root)
+    category_dir = out_root / _slug(category_label)
+    category_dir.mkdir(parents=True, exist_ok=True)
+    for chunk_id, text in sorted(texts.items()):
+        (category_dir / f"{_source_doc_id(chunk_id)}.md").write_text(
+            _concept_document(chunk_id, text, enrichment[chunk_id]), encoding="utf-8"
+        )
+    _write_category_index(category_dir)
+    _write_root_index(out_root, okf_version=okf_version, recipe_version=recipe_version)
+
+
+def register_okf_compile(registry: CapabilityRegistry) -> None:
+    """Register `okf_compile` (FR-K.1-K.4): an in-process `function`, category-3 (no ARD manifest)."""
+    registry.register(
+        "okf_compile",
+        contract=OkfBundleManifest,
+        kind="function",
+        display_name="OKF bundle compile (chunk-only, signpost-enriched)",
+    )
+
+
+# --- ACORD compile driver (the RAC verify entrypoint: uv run python -m rag_wright.okf.compile) ----
+
+_ACORD_SIDECAR = Path("data/acord/chunk_text")
+_ACORD_OUT = Path("data/acord/okf/bundle")
+_ACORD_ENRICH_CACHE = Path("data/acord/okf/enrichment.json")
+
+
+def _load_sidecar_texts(sidecar_root: Path) -> dict[str, str]:
+    """All ingested chunk texts as {chunk_id: text} by reading the T40 sidecar (one JSON file per doc)."""
+    texts: dict[str, str] = {}
+    for path in sorted(sidecar_root.glob("*.json")):
+        texts.update(json.loads(path.read_text(encoding="utf-8")))
+    return texts
+
+
+def _fallback_enrichment(chunk_id: str, text: str) -> EnrichedClause:
+    """Deterministic enrichment for a clause the model could not classify: uncategorized, first-sentence desc.
+
+    Guarantees every ingested clause gets a bundle file (RAC-46) even when the cheap model persistently fails
+    on it. These land in `_uncategorized/` and their count is reported, so the fallback is visible, not silent.
+    """
+    first = " ".join(text.split())[:120]
+    return EnrichedClause(chunk_id=chunk_id, category="", description=first or "(no description)", categorized=False)
+
+
+def _load_env() -> None:
+    """Load `.env` into the environment for the CLI run (the seam reads OPENROUTER_* from os.environ).
+
+    Minimal parser mirroring conftest, so no new dependency and no reliance on the pytest env hook.
+    """
+    import os
+
+    env = Path(".env")
+    if not env.exists():
+        return
+    for line in env.read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        os.environ.setdefault(key.strip(), value.strip())
+
+
+def main() -> None:
+    import asyncio
+
+    from rag_wright.okf.enrich import (
+        SeamClassifier,
+        enrich_all,
+        load_enrich_cache,
+        save_enrich_cache,
+    )
+    from rag_wright.okf.lint import lint_bundle
+
+    _load_env()
+    texts = _load_sidecar_texts(_ACORD_SIDECAR)
+    print(f"[okf_compile] {len(texts)} ingested clauses from {_ACORD_SIDECAR}")
+
+    cache = load_enrich_cache(_ACORD_ENRICH_CACHE, RECIPE_VERSION)
+    enrichment: dict[str, EnrichedClause] = cache
+    if len(enrichment) < len(texts):
+        SeamClassifier()(next(iter(texts.values())))  # preflight: fail loud on a systemic error (bad key/model)
+    for attempt in range(1, 8):  # resumable: each pass gates on the cache, retries only what is still missing
+        before = len(enrichment)
+        enrichment = asyncio.run(enrich_all(texts, SeamClassifier(), concurrency=12, cache=enrichment))
+        save_enrich_cache(_ACORD_ENRICH_CACHE, RECIPE_VERSION, enrichment)  # LLM results only (no fallbacks)
+        print(f"[okf_compile] enriched {len(enrichment)}/{len(texts)} (pass {attempt})")
+        if len(enrichment) == len(texts) or len(enrichment) == before:
+            break  # done, or a pass made no progress -> the residue goes to the deterministic fallback
+
+    missing = [cid for cid in texts if cid not in enrichment]
+    if missing:
+        if len(enrichment) < len(texts) // 2:  # a majority failing is systemic, not a stubborn tail
+            raise RuntimeError(
+                f"only {len(enrichment)}/{len(texts)} clauses enriched by the model; aborting (systemic issue)"
+            )
+        print(f"[okf_compile] {len(missing)} clauses fell back to deterministic (uncategorized) enrichment")
+        for cid in missing:  # every ingested clause must get a bundle file (RAC-46); fallbacks are not cached
+            enrichment[cid] = _fallback_enrichment(cid, texts[cid])
+
+    manifest = compile_bundle(texts, enrichment, _ACORD_OUT)
+    print(f"[okf_compile] bundle: {manifest.n_clauses} clauses, {manifest.n_categorized} categorized")
+    for label, n in manifest.categories.items():
+        print(f"             {n:4d}  {label}")
+    report = lint_bundle(_ACORD_OUT)
+    print(f"[okf_compile] lint: passes={report.passes} | orphan_rate={report.orphan_rate:.3f} | "
+          f"description_coverage={report.description_coverage:.3f} | "
+          f"broken_link_ratio={report.broken_link_ratio:.3f}")
+    print(f"[okf_compile] wrote {_ACORD_OUT}")
+
+
+if __name__ == "__main__":
+    main()

rag_wright/okf/document.py

@@ -0,0 +1,47 @@
+"""OKF concept-document (de)serialization: YAML frontmatter + markdown body.
+
+Mirrors the OKF reference `OKFDocument` (knowledge-catalog/okf) on our stack: a document is a `---`
+delimited YAML frontmatter block followed by a markdown body. `serialize_okf` preserves key order and
+appends the body verbatim (so a chunk body stays byte-faithful to its sidecar text); `parse_okf`
+round-trips it. YAML handles the escaping of descriptions that carry colons, quotes, or brackets, so
+the messy-real-text failure mode does not reach the frontmatter.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import yaml
+
+_DELIM = "---"
+
+
+class OkfParseError(ValueError):
+    """A concept document whose frontmatter is unterminated or is not a YAML mapping."""
+
+
+def serialize_okf(frontmatter: dict[str, Any], body: str) -> str:
+    """Serialize a concept document: `---` frontmatter (key order preserved) then the body verbatim."""
+    fm_text = yaml.safe_dump(frontmatter, sort_keys=False, allow_unicode=True).rstrip()
+    body = body if body.endswith("\n") else body + "\n"
+    return f"{_DELIM}\n{fm_text}\n{_DELIM}\n\n{body}"
+
+
+def parse_okf(text: str) -> tuple[dict[str, Any], str]:
+    """Parse a concept document into (frontmatter, body). No frontmatter -> ({}, whole text)."""
+    lines = text.splitlines()
+    if not lines or lines[0].strip() != _DELIM:
+        return {}, text
+    end = next((i for i in range(1, len(lines)) if lines[i].strip() == _DELIM), None)
+    if end is None:
+        raise OkfParseError("unterminated YAML frontmatter block")
+    try:
+        fm = yaml.safe_load("\n".join(lines[1:end])) or {}
+    except yaml.YAMLError as e:
+        raise OkfParseError(f"invalid YAML in frontmatter: {e}") from e
+    if not isinstance(fm, dict):
+        raise OkfParseError("frontmatter must be a YAML mapping")
+    body = "\n".join(lines[end + 1 :])
+    if body.startswith("\n"):
+        body = body[1:]
+    return fm, body

rag_wright/okf/enrich.py

@@ -0,0 +1,176 @@
+"""OKF signpost enrichment (FR-K.2, T46): category + one-line description per clause.
+
+The one model-bearing step of the compile. For each clause it produces the two signposts a traversal
+filters on without reading a body: the category (drives the directory tree and `tags`) and a one-line
+description (the `index.md` entry text). ACORD ships no clause categories and its stored "summary" is the
+full clause text, so both are manufactured here by a corpus-appropriate classifier through the
+model-profile seam, NOT by `graph_extraction`'s fixed CUAD ontology (ADR-0022: 41-CUAD covers only 67% of
+ACORD gold; a direct ACORD-9 classifier agrees 91.6%).
+
+Model: a cheap model for this simple task ONLY (the `OKF_ENRICHMENT` profile role, ADR-0023). Everything
+else in the system stays on its DeepSeek/Gemma role. The call is content-hash gated (keyed by `chunk_id`,
+which embeds the
+content hash) and concurrent (async + semaphore), per the parallel-LLM rule.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from enum import Enum
+from pathlib import Path
+from typing import Optional, Protocol, runtime_checkable
+
+from pydantic import BaseModel
+
+from rag_wright.models.profiles import ModelRole, model_for
+from rag_wright.models.seam import build_structured
+
+# ACORD's own 9 attorney categories (the corpus-appropriate label set; ADR-0022). A different corpus
+# supplies its own list.
+ACORD_CATEGORIES = [
+    "Limitation of Liability",
+    "Indemnification",
+    "Restrictive Covenants",
+    "Governing Law",
+    "Affirmative Covenants",
+    "IP Ownership/License",
+    "Term",
+    "Liquidated Damages",
+    "third party beneficiary clause",
+]
+_NONE = "None of these"
+
+
+class AcordCat(str, Enum):
+    LOL = "Limitation of Liability"
+    INDEMN = "Indemnification"
+    RESTRICT = "Restrictive Covenants"
+    GOVLAW = "Governing Law"
+    AFFIRM = "Affirmative Covenants"
+    IP = "IP Ownership/License"
+    TERM = "Term"
+    LIQDAM = "Liquidated Damages"
+    TPB = "third party beneficiary clause"
+    NONE = _NONE
+
+
+class ClauseEnrichment(BaseModel):
+    """The structured classifier output: one category (or 'None of these') and a one-line description."""
+
+    category: AcordCat
+    description: str
+
+
+class EnrichedClause(BaseModel):
+    """One clause's signposts, keyed by chunk_id. `categorized` is False when the classifier abstained."""
+
+    chunk_id: str
+    category: str
+    description: str
+    categorized: bool
+
+
+@runtime_checkable
+class ClauseClassifier(Protocol):
+    """Text -> {category, description}. The seam a test stubs so the compile is exercised without a model."""
+
+    def __call__(self, text: str) -> ClauseEnrichment: ...
+
+
+def enrichment_prompt(text: str, categories: list[str] = ACORD_CATEGORIES) -> str:
+    """The classify-and-describe prompt (shared by the real classifier and the bench, so they match)."""
+    return (
+        "You are enriching a contract clause for a knowledge index.\n"
+        "1) Classify it into exactly ONE category (or 'None of these' if none fit).\n"
+        "2) Write a ONE-LINE description (<= 15 words) that a lawyer could use to tell this clause apart "
+        "from others of the same category. Describe what THIS clause specifically says, not the category.\n\n"
+        "Categories:\n- " + "\n- ".join(categories) + "\n\n"
+        "Clause:\n" + text[:2000]
+    )
+
+
+class SeamClassifier:
+    """The real classifier: structured output through the model-profile seam on the OKF_ENRICHMENT role.
+
+    Retries a few times because a cheap model occasionally emits no valid tool call, which
+    `with_structured_output` surfaces as a `None` return (not an exception); a bare `None` is a transient
+    miss, so retry, and only raise when it persists (the caller skips a persistent failure and re-gates it).
+    """
+
+    def __init__(
+        self, model_id: Optional[str] = None, categories: list[str] = ACORD_CATEGORIES, *, retries: int = 3
+    ) -> None:
+        self._runnable = build_structured(model_id or model_for(ModelRole.OKF_ENRICHMENT), ClauseEnrichment)
+        self._categories = categories
+        self._retries = retries
+
+    def __call__(self, text: str) -> ClauseEnrichment:
+        prompt = enrichment_prompt(text, self._categories)
+        last_error: Exception | None = None
+        for _ in range(self._retries):
+            try:
+                v = self._runnable.invoke(prompt)
+            except Exception as e:  # noqa: BLE001 - transient provider/parse error; retry
+                last_error = e
+                continue
+            if v is not None:
+                return v
+        raise last_error or ValueError("no structured output after retries")
+
+
+async def enrich_all(
+    texts: dict[str, str],
+    classify: ClauseClassifier,
+    *,
+    concurrency: int = 8,
+    cache: Optional[dict[str, EnrichedClause]] = None,
+) -> dict[str, EnrichedClause]:
+    """Enrich every chunk, reusing `cache` (content-hash gated by chunk_id) and classifying only the rest.
+
+    Concurrent (async + semaphore) over the un-cached clauses. Determinism holds: the returned map is keyed
+    by chunk_id regardless of completion order.
+    """
+    cache = cache or {}
+    todo = {cid: text for cid, text in texts.items() if cid not in cache}
+    sem = asyncio.Semaphore(concurrency)
+
+    async def one(chunk_id: str, text: str) -> tuple[str, Optional[EnrichedClause]]:
+        async with sem:
+            try:
+                v = await asyncio.to_thread(classify, text)
+            except Exception:  # noqa: BLE001 - a rate-limited/failed call is skipped, not fatal;
+                return chunk_id, None  # it stays un-cached so a gated re-run retries it
+        if v is None:  # a classifier that yields no structured output is a skip, not a crash
+            return chunk_id, None
+        categorized = v.category != AcordCat.NONE
+        return chunk_id, EnrichedClause(
+            chunk_id=chunk_id,
+            category=v.category.value if categorized else "",
+            description=v.description,
+            categorized=categorized,
+        )
+
+    fresh = {cid: ec for cid, ec in await asyncio.gather(*(one(c, t) for c, t in todo.items())) if ec}
+    merged = {**cache, **fresh}
+    return {cid: merged[cid] for cid in texts if cid in merged}  # excludes clauses that failed
+
+
+def load_enrich_cache(path: Path, recipe_version: str) -> dict[str, EnrichedClause]:
+    """Load the enrichment cache if it matches `recipe_version`; a recipe change invalidates it."""
+    if not path.exists():
+        return {}
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if data.get("recipe_version") != recipe_version:
+        return {}
+    return {cid: EnrichedClause.model_validate(e) for cid, e in data.get("entries", {}).items()}
+
+
+def save_enrich_cache(path: Path, recipe_version: str, entries: dict[str, EnrichedClause]) -> None:
+    """Persist the enrichment cache keyed by recipe_version (the content-hash gate's durable half)."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {
+        "recipe_version": recipe_version,
+        "entries": {cid: e.model_dump() for cid, e in entries.items()},
+    }
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")