okf-core 0.3.0__py3-none-any.whl

okf_core/__init__.py

@@ -0,0 +1,110 @@
+"""okf-core: CanonicalDoc, OKF v0.2 parser/writer, five-level lint, pipeline,
+revisions, packer.
+
+The heart of OpenKnoll — the CLI and the cloud pipeline job run this identical
+code.
+"""
+
+from okf_core._version import __version__
+from okf_core.ask import AskResult, Citation, answer_question, write_trace
+from okf_core.cache import BuildCache
+from okf_core.canonical import (
+    Anchor,
+    Block,
+    CanonicalDoc,
+    Link,
+    MediaRef,
+    SourceRef,
+    sha256_hex,
+)
+from okf_core.explorer import Explorer, ExplorerError
+from okf_core.findings import Finding, Level, LintReport, Severity
+from okf_core.frontmatter import (
+    KNOWN_KEY_ORDER,
+    STATUS_VALUES,
+    Frontmatter,
+    FrontmatterError,
+    FrontmatterShapeError,
+    FrontmatterYamlError,
+    ParsedDocument,
+    parse_document,
+    write_document,
+)
+from okf_core.indexing import build_link_graph, ensure_index, search_index, write_index
+from okf_core.lint import LintConfig, lint_bundle
+from okf_core.packer import PackResult, pack_bundle, strip_okf_fields
+from okf_core.pipeline import (
+    BuildOutcome,
+    PipelineError,
+    PipelineSource,
+    build_revision,
+    check_reproducibility,
+)
+from okf_core.provider import (
+    GENERATOR_VERSION,
+    ModelProvider,
+    StubModelProvider,
+    generation_cache_key,
+    resolve_provider,
+)
+from okf_core.rag import EmbeddingProvider, StubEmbeddingProvider, resolve_embedder
+from okf_core.revision import read_current_revision_id, revision_dir
+from okf_core.viz import build_graph, render_html, write_viz
+
+__all__ = [
+    "GENERATOR_VERSION",
+    "KNOWN_KEY_ORDER",
+    "STATUS_VALUES",
+    "Anchor",
+    "AskResult",
+    "Block",
+    "BuildCache",
+    "BuildOutcome",
+    "CanonicalDoc",
+    "Citation",
+    "EmbeddingProvider",
+    "Explorer",
+    "ExplorerError",
+    "Finding",
+    "Frontmatter",
+    "FrontmatterError",
+    "FrontmatterShapeError",
+    "FrontmatterYamlError",
+    "Level",
+    "Link",
+    "LintConfig",
+    "LintReport",
+    "MediaRef",
+    "ModelProvider",
+    "PackResult",
+    "ParsedDocument",
+    "PipelineError",
+    "PipelineSource",
+    "Severity",
+    "SourceRef",
+    "StubEmbeddingProvider",
+    "StubModelProvider",
+    "__version__",
+    "answer_question",
+    "build_graph",
+    "build_link_graph",
+    "build_revision",
+    "check_reproducibility",
+    "ensure_index",
+    "generation_cache_key",
+    "lint_bundle",
+    "pack_bundle",
+    "parse_document",
+    "read_current_revision_id",
+    "render_html",
+    "resolve_embedder",
+    "resolve_provider",
+    "revision_dir",
+    "search_index",
+    "sha256_hex",
+    "strip_okf_fields",
+    "write_document",
+    "write_index",
+    "write_trace",
+    "write_viz",
+]

okf_core/_version.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

okf_core/ask.py

@@ -0,0 +1,512 @@
+"""One-shot explorer answers: navigation policy, answer contract, trace.
+
+Navigation policy is deterministic code, not model output: start at
+``overview``, narrow with lexical ``search``, ``peek`` before ``read``, follow
+``links`` with bounded fan-out (≤4), stop when the answer is supported or the
+token budget (default 25,000 input tokens) is spent. The model only writes the
+final prose over retrieved evidence — with the deterministic stub the whole
+answer is a pure function of the bundle and the question.
+
+Answer contract: citations carry qualified concept paths plus OKF source
+ids and resource links; warnings flag draft/deprecated/stale/unverified
+evidence; on insufficient evidence the answer abstains and names what would
+resolve the gap. Bundle text is untrusted data — it is quoted, never obeyed.
+
+The trace is a separate record (tools, paths, retrieved characters, token
+estimates, latency, model, revision id, retrieval condition), derived state
+stored under ``.oknoll/traces/`` — never part of a portable bundle.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from okf_core import bundle as bundle_mod
+from okf_core.canonical import sha256_hex
+from okf_core.explorer import Explorer, estimate_tokens
+from okf_core.frontmatter import Frontmatter, parse_document
+from okf_core.indexing import question_terms, term_occurrences, tokenize
+from okf_core.provider import ModelProvider
+from okf_core.rag import (
+    CHUNK_CHARS,
+    EmbeddingProvider,
+    ensure_rag_index,
+    resolve_embedder,
+    retrieve,
+)
+from okf_core.revision import read_current_revision_id
+
+DEFAULT_TOKEN_BUDGET = 25_000
+MAX_LINK_FANOUT = 4
+MAX_CONCEPT_READS = 4
+MAX_EVIDENCE = 4
+EXCERPT_CHARS = 500
+TRACES_DIR = f"{bundle_mod.DERIVED_STATE_DIR}/traces"
+TRACE_SCHEMA_VERSION = 1
+
+
+@dataclass(frozen=True, slots=True)
+class Citation:
+    path: str  # qualified bundle path
+    title: str
+    source_ids: list[str]
+    resources: list[str]  # reference snapshot paths or original resource URIs
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "path": self.path,
+            "title": self.title,
+            "source_ids": self.source_ids,
+            "resources": self.resources,
+        }
+
+
+@dataclass(slots=True)
+class AskResult:
+    question: str
+    answer: str
+    abstained: bool
+    citations: list[Citation]
+    warnings: list[str]
+    trace: dict[str, Any]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "question": self.question,
+            "answer": self.answer,
+            "abstained": self.abstained,
+            "citations": [c.to_dict() for c in self.citations],
+            "warnings": self.warnings,
+            "trace": self.trace,
+        }
+
+
+@dataclass(slots=True)
+class _Traced:
+    """Wraps the explorer so every tool call lands in the trace with its cost."""
+
+    explorer: Explorer
+    events: list[dict[str, Any]] = field(default_factory=list)
+    spent_chars: int = 0
+
+    @property
+    def spent_tokens(self) -> int:
+        return estimate_tokens(self.spent_chars)
+
+    def call(self, tool: str, /, **kwargs: Any) -> dict[str, Any]:
+        method: Callable[..., dict[str, Any]] = getattr(self.explorer, tool)
+        result = method(**kwargs)
+        chars = len(json.dumps(result, ensure_ascii=False, default=str))
+        self.spent_chars += chars
+        self.events.append(
+            {"tool": tool, "args": kwargs, "chars": chars, "tokens": estimate_tokens(chars)}
+        )
+        return result
+
+
+_FOOTNOTE_MARKER_RE = re.compile(r"\[\^[^\]]+\]")
+
+
+def _paragraphs(body: str) -> list[str]:
+    return [p.strip() for p in body.split("\n\n") if p.strip()]
+
+
+def _best_excerpt(body: str, terms: set[str]) -> tuple[str, int]:
+    """The paragraph with the most distinct question-term hits.
+
+    Ties break on total term occurrences, then earliest. A single-salient-term
+    question ("What is A2K?") gives every mentioning paragraph the same distinct
+    count; occurrence density is what separates the section that answers it from
+    a passing mention, and it is just as deterministic.
+    """
+    best, best_key = "", (0, 0)
+    for para in _paragraphs(body):
+        distinct = len(terms & tokenize(para))
+        if distinct == 0:
+            continue
+        key = (distinct, term_occurrences(para, terms))
+        if key > best_key:
+            best, best_key = para, key
+    excerpt = _FOOTNOTE_MARKER_RE.sub("", best).strip()  # markers are for the file, not prose
+    return excerpt[:EXCERPT_CHARS], best_key[0]
+
+
+def _concept_warnings(path: str, frontmatter: dict[str, Any], today: str | None) -> list[str]:
+    """Trust warnings for one cited concept, phrased next to the affected claim."""
+    fm = Frontmatter(data=frontmatter)
+    warnings: list[str] = []
+    if fm.status == "draft":
+        warnings.append(f"{path}: status is draft — content is unreviewed")
+    elif fm.status == "deprecated":
+        warnings.append(f"{path}: status is deprecated — may no longer apply")
+    if today and fm.stale_after and fm.stale_after < today:
+        warnings.append(f"{path}: stale since {fm.stale_after}")
+    if not fm.verified:
+        warnings.append(f"{path}: unverified — no verification record")
+    return warnings
+
+
+def _citation_for(path: str, frontmatter: dict[str, Any], title: str) -> Citation:
+    fm = Frontmatter(data=frontmatter)
+    resources: list[str] = []
+    for source in fm.sources:
+        resource = source.get("resource")
+        if isinstance(resource, str):
+            resources.append(resource)
+    origin = frontmatter.get("openknoll_source")
+    if isinstance(origin, dict) and isinstance(origin.get("uri"), str):
+        resources.append(origin["uri"])  # reference snapshots cite their origin
+    return Citation(path=path, title=title, source_ids=fm.source_ids(), resources=resources)
+
+
+def answer_question(
+    *,
+    bundle_dir: Path,
+    question: str,
+    provider: ModelProvider,
+    today: str | None = None,
+    budget_tokens: int = DEFAULT_TOKEN_BUDGET,
+    condition: str = "pd",
+    embedder: EmbeddingProvider | None = None,
+    clock: Callable[[], str] | None = None,
+    timer: Callable[[], float] | None = None,
+) -> AskResult:
+    """Answer one question under a retrieval condition.
+
+    ``pd`` runs the deterministic navigation policy over concepts; ``rag`` runs
+    the vector top-k baseline over the identical normalized corpus (reference
+    snapshots). Both share the answer contract, the evidence budget
+    (≤ MAX_EVIDENCE excerpts of ≤ EXCERPT_CHARS), and the trace shape.
+    """
+    if condition not in ("pd", "rag"):
+        raise ValueError(f"unknown retrieval condition {condition!r} — available: 'pd', 'rag'")
+    clock = clock or (lambda: datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ"))
+    timer = timer or time.monotonic
+    if condition == "rag":
+        return _answer_rag(
+            bundle_dir=bundle_dir,
+            question=question,
+            provider=provider,
+            embedder=embedder if embedder is not None else resolve_embedder("stub"),
+            budget_tokens=budget_tokens,
+            clock=clock,
+            timer=timer,
+        )
+    started_at = clock()
+    t0 = timer()
+
+    explorer = Explorer(bundle_dir, today=today)
+    traced = _Traced(explorer)
+    terms = set(question_terms(question))
+
+    # 1. overview is always the first call (navigation-policy guardrail).
+    traced.call("overview")
+
+    # 2. narrow through lexical search. Concepts are the navigation surface;
+    #    reference snapshots are raw sources, consulted only when no concept hit.
+    search = traced.call("search", query=question)
+    results = list(search["results"])
+    concept_hits = [str(r["path"]) for r in results if r["kind"] == "concept"]
+    reference_hits = [str(r["path"]) for r in results if r["kind"] == "reference"]
+    candidates = concept_hits[:MAX_CONCEPT_READS]
+    if not candidates:
+        candidates = reference_hits[:MAX_CONCEPT_READS]
+
+    # 3. peek before read; both stay within the token budget — peeks over large
+    #    bundles are not free, and the budget is a hard cap, not a suggestion.
+    read_docs: list[dict[str, Any]] = []
+    budget_exhausted = False
+    for path in candidates:
+        if traced.spent_tokens >= budget_tokens:
+            budget_exhausted = True
+            break
+        traced.call("peek", path=path)
+    for path in candidates:
+        if traced.spent_tokens >= budget_tokens:
+            budget_exhausted = True
+            break
+        doc = traced.call("read", path=path)
+        doc["_terms"] = terms
+        doc["_via"] = None
+        read_docs.append(doc)
+
+    # 4. follow concept→concept links with bounded fan-out (≤4). A hop is
+    #    justified by its anchor text, so the linked document is scored against
+    #    the question terms *plus* that anchor's terms.
+    followed = 0
+    seen_paths = {str(doc["path"]) for doc in read_docs}
+    for doc in list(read_docs):
+        if followed >= MAX_LINK_FANOUT or traced.spent_tokens >= budget_tokens:
+            break
+        parent = str(doc["path"])
+        if not bundle_mod.is_concept(parent):
+            continue
+        anchors = {
+            str(link["resolved"]): str(link["text"])
+            for link in doc["links"]
+            if isinstance(link, dict) and link.get("resolved")
+        }
+        edges = traced.call("links", path=parent)
+        for target in list(edges["outbound"]):
+            if followed >= MAX_LINK_FANOUT:
+                break
+            if traced.spent_tokens >= budget_tokens:
+                budget_exhausted = True
+                break
+            if target in seen_paths or not bundle_mod.is_concept(target):
+                continue
+            seen_paths.add(target)
+            linked = traced.call("read", path=target)
+            linked["_terms"] = terms | tokenize(anchors.get(target, ""))
+            linked["_via"] = parent
+            read_docs.append(linked)
+            followed += 1
+
+    # 5. deterministic evidence selection: best matching paragraph per document,
+    #    with reference snapshots of an already-cited concept folded away.
+    evidence: list[dict[str, Any]] = []
+    for doc in read_docs:
+        doc_terms = doc["_terms"] if isinstance(doc["_terms"], set) else terms
+        excerpt, score = _best_excerpt(str(doc["body"]), doc_terms)
+        if score < 1:
+            continue
+        frontmatter = doc["frontmatter"] if isinstance(doc["frontmatter"], dict) else {}
+        title = frontmatter.get("title") or str(doc["path"])
+        evidence.append(
+            {
+                "path": str(doc["path"]),
+                "title": str(title),
+                "excerpt": excerpt,
+                "score": score,
+                "frontmatter": frontmatter,
+                "via": doc["_via"],
+            }
+        )
+    evidence.sort(key=lambda e: (-int(e["score"]), str(e["path"])))
+    cited_resources: set[str] = set()
+    for entry in evidence:
+        for source in Frontmatter(data=entry["frontmatter"]).sources:
+            resource = source.get("resource")
+            if isinstance(resource, str):
+                cited_resources.add(resource)
+    evidence = [e for e in evidence if e["path"] not in cited_resources][:MAX_EVIDENCE]
+
+    citations: list[Citation] = []
+    warnings: list[str] = []
+    if not evidence:
+        abstained = True
+        wanted = ", ".join(sorted(terms)) or "the question topic"
+        if candidates:
+            answer = (
+                "Insufficient evidence in this bundle to answer. "
+                f"The closest match is {candidates[0]}, but it does not address: {wanted}. "
+                f"A concept covering {wanted} would resolve the gap."
+            )
+        else:
+            answer = (
+                "Insufficient evidence in this bundle to answer. "
+                f"No concept mentions: {wanted}. "
+                f"Adding a source about {wanted} would resolve the gap."
+            )
+    else:
+        abstained = False
+        answer = provider.complete(
+            "answer-question",
+            {
+                "question": question,
+                "evidence": [
+                    {"path": e["path"], "title": e["title"], "excerpt": e["excerpt"]}
+                    for e in evidence
+                ],
+            },
+        )
+        for entry in evidence:
+            frontmatter = entry["frontmatter"]
+            citations.append(_citation_for(entry["path"], frontmatter, entry["title"]))
+            if bundle_mod.is_concept(entry["path"]):
+                warnings.extend(_concept_warnings(entry["path"], frontmatter, today))
+
+    latency_ms = int((timer() - t0) * 1000)
+    trace: dict[str, Any] = {
+        "schema_version": TRACE_SCHEMA_VERSION,
+        "question": question,
+        "condition": condition,
+        "model": provider.id,
+        "revision_id": read_current_revision_id(explorer.root),
+        "started_at": started_at,
+        "latency_ms": latency_ms,
+        "budget": {
+            "limit_tokens": budget_tokens,
+            "spent_tokens": traced.spent_tokens,
+            "spent_chars": traced.spent_chars,
+            "exhausted": budget_exhausted,
+        },
+        "policy": {
+            "max_link_fanout": MAX_LINK_FANOUT,
+            "max_concept_reads": MAX_CONCEPT_READS,
+        },
+        "tools": traced.events,
+        "paths_read": sorted({str(doc["path"]) for doc in read_docs}),
+        "hops": [
+            {"path": str(doc["path"]), "via": doc["_via"]}
+            for doc in read_docs
+            if doc["_via"] is not None
+        ],
+        "evidence_paths": [str(e["path"]) for e in evidence],
+        "abstained": abstained,
+    }
+
+    return AskResult(
+        question=question,
+        answer=answer,
+        abstained=abstained,
+        citations=citations,
+        warnings=warnings,
+        trace=trace,
+    )
+
+
+def _answer_rag(
+    *,
+    bundle_dir: Path,
+    question: str,
+    provider: ModelProvider,
+    embedder: EmbeddingProvider,
+    budget_tokens: int,
+    clock: Callable[[], str],
+    timer: Callable[[], float],
+) -> AskResult:
+    """The vector top-k baseline.
+
+    Retrieval is embedding-only over reference-snapshot chunks — no index-first
+    navigation, no link following, and no concept trust metadata, which is
+    exactly what the PD-vs-RAG comparison measures (RQ3: references carry no
+    status/verification fields, so the warnings channel is structurally empty).
+    """
+    started_at = clock()
+    t0 = timer()
+
+    index = ensure_rag_index(bundle_dir, embedder)
+    [query_vector] = embedder.embed([question])
+    hits = retrieve(index, query_vector, MAX_EVIDENCE)
+
+    # Equal budgets with PD: evidence is capped per-passage by construction
+    # (CHUNK_CHARS == EXCERPT_CHARS) and total retrieved chars are budgeted.
+    events: list[dict[str, Any]] = []
+    spent_chars = len(question)
+    selected: list[dict[str, Any]] = []
+    budget_exhausted = False
+    for hit in hits:
+        if estimate_tokens(spent_chars + len(hit["text"])) > budget_tokens:
+            budget_exhausted = True
+            break
+        spent_chars += len(hit["text"])
+        selected.append(hit)
+    events.append(
+        {
+            "tool": "vector_search",
+            "args": {"k": MAX_EVIDENCE, "embedder": embedder.id, "chunks": len(index["chunks"])},
+            "chars": spent_chars,
+            "tokens": estimate_tokens(spent_chars),
+        }
+    )
+
+    frontmatters: dict[str, dict[str, Any]] = {}
+    evidence: list[dict[str, Any]] = []
+    for hit in selected:
+        path = str(hit["path"])
+        if path not in frontmatters:
+            parsed = parse_document((bundle_dir / path).read_text(encoding="utf-8"))
+            frontmatters[path] = parsed.frontmatter.data if parsed.frontmatter else {}
+        title = frontmatters[path].get("title") or path
+        evidence.append({"path": path, "title": str(title), "excerpt": str(hit["text"])})
+
+    citations: list[Citation] = []
+    if not evidence:
+        abstained = True
+        wanted = ", ".join(sorted(set(question_terms(question)))) or "the question topic"
+        answer = (
+            "Insufficient evidence in this bundle to answer. "
+            f"No indexed passage matches: {wanted}. "
+            f"Adding a source about {wanted} would resolve the gap."
+        )
+    else:
+        abstained = False
+        answer = provider.complete(
+            "answer-question",
+            {
+                "question": question,
+                "evidence": [
+                    {"path": e["path"], "title": e["title"], "excerpt": e["excerpt"]}
+                    for e in evidence
+                ],
+            },
+        )
+        for path in dict.fromkeys(str(e["path"]) for e in evidence):
+            frontmatter = frontmatters[path]
+            title = str(frontmatter.get("title") or path)
+            citations.append(_citation_for(path, frontmatter, title))
+
+    latency_ms = int((timer() - t0) * 1000)
+    trace: dict[str, Any] = {
+        "schema_version": TRACE_SCHEMA_VERSION,
+        "question": question,
+        "condition": "rag",
+        "model": provider.id,
+        "revision_id": index["revision_id"],
+        "started_at": started_at,
+        "latency_ms": latency_ms,
+        "budget": {
+            "limit_tokens": budget_tokens,
+            "spent_tokens": estimate_tokens(spent_chars),
+            "spent_chars": spent_chars,
+            "exhausted": budget_exhausted,
+        },
+        "policy": {
+            "k": MAX_EVIDENCE,
+            "chunk_chars": CHUNK_CHARS,
+            "embedder": embedder.id,
+            "index_chunks": len(index["chunks"]),
+        },
+        "tools": events,
+        "paths_read": sorted({str(e["path"]) for e in evidence}),
+        "hops": [],
+        "evidence_paths": [str(e["path"]) for e in evidence],
+        "abstained": abstained,
+    }
+
+    return AskResult(
+        question=question,
+        answer=answer,
+        abstained=abstained,
+        citations=citations,
+        # References carry no status/stale/verified metadata, so RAG cannot
+        # flag draft/deprecated/stale evidence — a measured property, not a bug.
+        warnings=[],
+        trace=trace,
+    )
+
+
+def write_trace(bundle_dir: Path, result: AskResult) -> Path:
+    """Persist the trace as derived state under ``.oknoll/traces/``."""
+    traces = bundle_dir / TRACES_DIR
+    traces.mkdir(parents=True, exist_ok=True)
+    stamp = str(result.trace.get("started_at", "")).replace(":", "").replace("-", "")
+    digest = sha256_hex(result.question)[:8]
+    base = f"ask-{stamp}-{digest}"
+    path = traces / f"{base}.json"
+    counter = 2
+    while path.exists():
+        path = traces / f"{base}-{counter}.json"
+        counter += 1
+    path.write_text(
+        json.dumps(result.to_dict(), indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
+    )
+    return path