docsgraph 0.1.0a2__py3-none-any.whl

cairn/embed/openai_compatible.py

@@ -0,0 +1,155 @@
+"""OpenAI-compatible HTTP embedder.
+
+Works with any endpoint that implements the OpenAI ``/v1/embeddings``
+contract: OpenAI itself, Ollama (``http://localhost:11434/v1``), vLLM,
+Together, Anyscale, etc.
+
+Default configuration points at a local Ollama instance running the
+``nomic-embed-text`` model (768 dims) — chosen for the same reason as the
+summarizer default: zero API keys, mature stack, runs on a laptop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import httpx
+
+from cairn.core.errors import IndexBuildError
+
+
+class OpenAICompatibleEmbedder:
+    """OpenAI-compatible embeddings client."""
+
+    def __init__(
+        self,
+        *,
+        base_url: str = "http://localhost:11434/v1",
+        model: str = "nomic-embed-text",
+        dim: int = 768,
+        api_key: str | None = None,
+        timeout: float = 60.0,
+        max_retries: int = 2,
+        retry_base_delay: float = 0.5,
+    ) -> None:
+        if dim < 1:
+            msg = f"dim must be >= 1; got {dim}"
+            raise ValueError(msg)
+        if max_retries < 0:
+            msg = f"max_retries must be >= 0; got {max_retries}"
+            raise ValueError(msg)
+        if retry_base_delay < 0:
+            msg = f"retry_base_delay must be >= 0; got {retry_base_delay}"
+            raise ValueError(msg)
+        self.base_url = base_url.rstrip("/")
+        self.model = model
+        self.dim = dim
+        self.api_key = api_key
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.retry_base_delay = retry_base_delay
+        self.name = f"openai-compat:{model}"
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        payload: dict[str, Any] = {
+            "model": self.model,
+            "input": texts,
+        }
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            response = await self._post_with_retries(client, payload, headers)
+            data = response.json()
+
+        try:
+            vectors = [list(item["embedding"]) for item in data["data"]]
+        except (KeyError, TypeError, IndexError) as exc:
+            msg = "embedder response did not match OpenAI embeddings shape"
+            raise IndexBuildError(msg, details={"response": data}) from exc
+
+        if len(vectors) != len(texts):
+            msg = (
+                f"embedder returned {len(vectors)} vectors for "
+                f"{len(texts)} inputs"
+            )
+            raise IndexBuildError(msg)
+        for i, vec in enumerate(vectors):
+            if len(vec) != self.dim:
+                msg = (
+                    f"embedder returned dim={len(vec)} but client expects "
+                    f"dim={self.dim} (model {self.model!r}, index {i})"
+                )
+                raise IndexBuildError(msg)
+
+        return vectors
+
+    async def _post_with_retries(
+        self,
+        client: httpx.AsyncClient,
+        payload: dict[str, Any],
+        headers: dict[str, str],
+    ) -> httpx.Response:
+        last_exc: httpx.HTTPError | None = None
+        for attempt in range(self.max_retries + 1):
+            try:
+                response = await client.post(
+                    f"{self.base_url}/embeddings",
+                    json=payload,
+                    headers=headers,
+                )
+            except httpx.HTTPError as exc:
+                last_exc = exc
+                if attempt < self.max_retries:
+                    await self._sleep_before_retry(attempt)
+                    continue
+                msg = f"embedder request failed: {exc}"
+                raise IndexBuildError(
+                    msg,
+                    details={
+                        "model": self.model,
+                        "base_url": self.base_url,
+                        "error_type": type(exc).__name__,
+                        "attempts": attempt + 1,
+                    },
+                ) from exc
+
+            if response.status_code in (429, 500, 502, 503, 504) and attempt < self.max_retries:
+                await self._sleep_before_retry(attempt)
+                continue
+            if response.status_code >= 400:
+                msg = (
+                    f"embedder endpoint returned HTTP {response.status_code}: "
+                    f"{response.text[:200]}"
+                )
+                raise IndexBuildError(
+                    msg,
+                    details={
+                        "status": response.status_code,
+                        "model": self.model,
+                        "base_url": self.base_url,
+                        "attempts": attempt + 1,
+                    },
+                )
+            return response
+
+        msg = "embedder request failed without a response"
+        raise IndexBuildError(
+            msg,
+            details={
+                "model": self.model,
+                "base_url": self.base_url,
+                "error_type": type(last_exc).__name__ if last_exc else None,
+            },
+        )
+
+    async def _sleep_before_retry(self, attempt: int) -> None:
+        if self.retry_base_delay == 0:
+            return
+        await asyncio.sleep(self.retry_base_delay * (2**attempt))

cairn/engine/__init__.py

@@ -0,0 +1,18 @@
+"""Engine layer — orchestrates the three sub-index builders + top-level manifest."""
+
+from cairn.engine.indexer import Indexer, IndexResult
+from cairn.engine.manifest import (
+    MANIFEST_FILENAME,
+    MANIFEST_FORMAT_VERSION,
+    Manifest,
+    read_manifest,
+)
+
+__all__ = [
+    "MANIFEST_FILENAME",
+    "MANIFEST_FORMAT_VERSION",
+    "IndexResult",
+    "Indexer",
+    "Manifest",
+    "read_manifest",
+]

cairn/engine/indexer.py

@@ -0,0 +1,298 @@
+"""Indexer — single entry point that builds all three v0.1 sub-indexes.
+
+Parses a source document, runs ``TreeBuilder`` synchronously, then
+``SummaryBuilder`` and ``VectorBuilder`` asynchronously, and finally writes
+the top-level ``manifest.json`` that ties the artifacts together.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from cairn import __version__
+from cairn.core.errors import IndexNotFoundError
+from cairn.core.types import Document
+from cairn.embed.base import Embedder
+from cairn.engine.manifest import (
+    MANIFEST_FILENAME,
+    MANIFEST_FORMAT_VERSION,
+    Manifest,
+    SubIndexEntry,
+    read_manifest,
+    write_manifest,
+)
+from cairn.entity.base import EntityExtractor
+from cairn.index.entities import (
+    ENTITIES_FILENAME,
+    ENTITIES_FORMAT_VERSION,
+    Entities,
+    EntityBuilder,
+)
+from cairn.index.summaries import (
+    SUMMARIES_FILENAME,
+    SUMMARIES_FORMAT_VERSION,
+    SummaryBuilder,
+)
+from cairn.index.tree import TREE_FILENAME, TreeBuilder
+from cairn.index.vectors import (
+    VECTORS_FORMAT_VERSION,
+    VECTORS_MANIFEST_FILENAME,
+    VectorBuilder,
+)
+from cairn.index.xrefs import (
+    XREFS_FILENAME,
+    XREFS_FORMAT_VERSION,
+    XRefBuilder,
+)
+from cairn.ingest.base import Parser
+from cairn.summarize.base import Summarizer, SummaryLevel
+from cairn.summarize.cache import SummaryCache
+from cairn.xref.base import XRefExtractor
+
+_TREE_BUILDER_VERSION = 1
+
+
+@dataclass(frozen=True)
+class IndexResult:
+    """Outcome of an :meth:`Indexer.index_path` call.
+
+    ``rebuilt`` is ``False`` when the source's hash matched the previous
+    build's manifest, all producer fingerprints still match, and the existing
+    index was kept as-is (a no-op).
+    """
+
+    manifest_path: Path
+    rebuilt: bool
+
+
+class Indexer:
+    """Orchestrates the sub-index builders for one document.
+
+    Tree + Summaries + Vectors are always built. The Entities sub-index is
+    built when ``entity_extractor`` is supplied (default since v0.2).
+    """
+
+    def __init__(
+        self,
+        *,
+        parser: Parser,
+        summarizer: Summarizer,
+        embedder: Embedder,
+        entity_extractor: EntityExtractor | None = None,
+        xref_extractor: XRefExtractor | None = None,
+        summary_cache: SummaryCache | None = None,
+        summary_concurrency: int = 4,
+        embed_batch_size: int = 32,
+        progress: Callable[[str], None] | None = None,
+    ) -> None:
+        self.parser = parser
+        self.summarizer = summarizer
+        self.embedder = embedder
+        self.entity_extractor = entity_extractor
+        self.xref_extractor = xref_extractor
+        self.summary_cache = summary_cache
+        self.summary_concurrency = summary_concurrency
+        self.embed_batch_size = embed_batch_size
+        self.progress = progress
+
+    async def index_path(
+        self,
+        source: Path,
+        *,
+        out_dir: Path,
+        doc_id: str | None = None,
+        summary_levels: Sequence[SummaryLevel] = (
+            SummaryLevel.GIST,
+            SummaryLevel.SYNOPSIS,
+            SummaryLevel.DIGEST,
+        ),
+        force: bool = False,
+    ) -> IndexResult:
+        """Parse a source file and build all sub-indexes.
+
+        When ``force`` is ``False`` (default), the indexer first checks
+        whether ``out_dir`` already contains a manifest whose source_hash and
+        producer fingerprints match the requested build. If so, the existing
+        index is left untouched and ``IndexResult.rebuilt`` is ``False``. Pass
+        ``force=True`` to always rebuild.
+        """
+        document = self.parser.parse(source, doc_id=doc_id)
+
+        if not force and _existing_matches(
+            out_dir,
+            document.source_hash,
+            summarizer=self.summarizer,
+            embedder=self.embedder,
+            summary_levels=summary_levels,
+            entity_extractor=self.entity_extractor,
+            xref_extractor=self.xref_extractor,
+        ):
+            return IndexResult(
+                manifest_path=out_dir / MANIFEST_FILENAME,
+                rebuilt=False,
+            )
+
+        manifest_path = await self.index_document(
+            document,
+            out_dir=out_dir,
+            summary_levels=summary_levels,
+        )
+        return IndexResult(manifest_path=manifest_path, rebuilt=True)
+
+    async def index_document(
+        self,
+        document: Document,
+        *,
+        out_dir: Path,
+        summary_levels: Sequence[SummaryLevel] = (
+            SummaryLevel.GIST,
+            SummaryLevel.SYNOPSIS,
+            SummaryLevel.DIGEST,
+        ),
+    ) -> Path:
+        """Run every configured builder against an already-parsed Document."""
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+        self._emit("tree: writing")
+        TreeBuilder().build(document, out_dir=out_dir)
+        self._emit("tree: done")
+        self._emit("summaries: starting")
+        await SummaryBuilder(
+            self.summarizer,
+            cache=self.summary_cache,
+            concurrency=self.summary_concurrency,
+            progress=lambda done, total: self._emit(f"summaries: {done}/{total}"),
+        ).build(document, out_dir=out_dir, levels=summary_levels)
+        self._emit("summaries: done")
+        self._emit("vectors: starting")
+        await VectorBuilder(
+            self.embedder, batch_size=self.embed_batch_size
+        ).build(document, out_dir=out_dir)
+        self._emit("vectors: done")
+
+        subindexes: dict[str, SubIndexEntry] = {
+            "tree": SubIndexEntry(
+                path=TREE_FILENAME,
+                builder_version=_TREE_BUILDER_VERSION,
+            ),
+            "summaries": SubIndexEntry(
+                path=SUMMARIES_FILENAME,
+                builder_version=SUMMARIES_FORMAT_VERSION,
+                model=self.summarizer.name,
+                levels=[lvl.value for lvl in summary_levels],
+            ),
+            "vectors": SubIndexEntry(
+                path=VECTORS_MANIFEST_FILENAME,
+                builder_version=VECTORS_FORMAT_VERSION,
+                embedder=self.embedder.name,
+                dim=self.embedder.dim,
+            ),
+        }
+
+        entities_reader: Entities | None = None
+        if self.entity_extractor is not None:
+            self._emit("entities: starting")
+            await EntityBuilder(self.entity_extractor).build(
+                document, out_dir=out_dir
+            )
+            self._emit("entities: done")
+            subindexes["entities"] = SubIndexEntry(
+                path=ENTITIES_FILENAME,
+                builder_version=ENTITIES_FORMAT_VERSION,
+                extractor=self.entity_extractor.name,
+            )
+            # Reload from disk so the xref extractor can use the canonical
+            # form of the just-built Entities sub-index.
+            entities_reader = Entities.load(out_dir)
+
+        if self.xref_extractor is not None:
+            self._emit("xrefs: starting")
+            await XRefBuilder(self.xref_extractor).build(
+                document, out_dir=out_dir, entities=entities_reader
+            )
+            self._emit("xrefs: done")
+            subindexes["xrefs"] = SubIndexEntry(
+                path=XREFS_FILENAME,
+                builder_version=XREFS_FORMAT_VERSION,
+                extractor=self.xref_extractor.name,
+            )
+
+        manifest = Manifest(
+            format_version=MANIFEST_FORMAT_VERSION,
+            doc_id=document.id,
+            cairn_version=__version__,
+            source_path=str(document.source_path),
+            source_hash=document.source_hash,
+            indexed_at=datetime.now(UTC),
+            subindexes=subindexes,
+        )
+        self._emit("manifest: writing")
+        path = write_manifest(out_dir, manifest)
+        self._emit("manifest: done")
+        return path
+
+    def _emit(self, message: str) -> None:
+        if self.progress is not None:
+            self.progress(message)
+
+
+def _existing_matches(
+    out_dir: Path,
+    source_hash: str,
+    *,
+    summarizer: Summarizer,
+    embedder: Embedder,
+    summary_levels: Sequence[SummaryLevel],
+    entity_extractor: EntityExtractor | None,
+    xref_extractor: XRefExtractor | None,
+) -> bool:
+    """Return ``True`` when the existing index matches source and producers."""
+    if not (out_dir / MANIFEST_FILENAME).exists():
+        return False
+    try:
+        existing = read_manifest(out_dir)
+    except IndexNotFoundError:
+        return False
+    if existing.source_hash != source_hash:
+        return False
+
+    tree = existing.subindexes.get("tree")
+    summaries = existing.subindexes.get("summaries")
+    vectors = existing.subindexes.get("vectors")
+    if tree is None or tree.builder_version != _TREE_BUILDER_VERSION:
+        return False
+    if (
+        summaries is None
+        or summaries.builder_version != SUMMARIES_FORMAT_VERSION
+        or summaries.model != summarizer.name
+        or summaries.levels != [lvl.value for lvl in summary_levels]
+    ):
+        return False
+    if (
+        vectors is None
+        or vectors.builder_version != VECTORS_FORMAT_VERSION
+        or vectors.embedder != embedder.name
+        or vectors.dim != embedder.dim
+    ):
+        return False
+
+    if entity_extractor is not None:
+        entities = existing.subindexes.get("entities")
+        if (
+            entities is None
+            or entities.builder_version != ENTITIES_FORMAT_VERSION
+            or entities.extractor != entity_extractor.name
+        ):
+            return False
+    if xref_extractor is not None:
+        xrefs = existing.subindexes.get("xrefs")
+        if (
+            xrefs is None
+            or xrefs.builder_version != XREFS_FORMAT_VERSION
+            or xrefs.extractor != xref_extractor.name
+        ):
+            return False
+    return True

cairn/engine/manifest.py

@@ -0,0 +1,83 @@
+"""Top-level document manifest.
+
+Per ARCHITECTURE.md §5, a document directory holds one ``manifest.json`` that
+records source provenance, sub-index file pointers, builder versions, and
+the model identifiers that produced each artifact. The manifest is the
+contract: any file it references must exist; orphans are reapable.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Final
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from cairn.core.errors import IndexNotFoundError
+
+MANIFEST_FILENAME: Final = "manifest.json"
+MANIFEST_FORMAT_VERSION: Final = 1
+
+
+class SubIndexEntry(BaseModel):
+    """One sub-index pointer in the top-level manifest."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    path: str = Field(description="path relative to the document directory")
+    builder_version: int = Field(ge=1)
+    # Optional fields that describe what produced this artifact.
+    model: str | None = None
+    embedder: str | None = None
+    extractor: str | None = None
+    dim: int | None = None
+    levels: list[str] | None = None
+
+
+class Manifest(BaseModel):
+    """Top-level document manifest — the contract for everything else."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    format_version: int
+    doc_id: str
+    cairn_version: str
+    source_path: str
+    source_hash: str
+    indexed_at: datetime
+    subindexes: dict[str, SubIndexEntry]
+
+
+def write_manifest(out_dir: Path, manifest: Manifest) -> Path:
+    """Write ``manifest.json`` into ``out_dir`` deterministically."""
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / MANIFEST_FILENAME
+
+    payload: dict[str, Any] = manifest.model_dump(mode="json")
+    with path.open("w", encoding="utf-8") as fh:
+        json.dump(payload, fh, ensure_ascii=False, indent=2)
+        fh.write("\n")
+    return path
+
+
+def read_manifest(doc_dir: Path) -> Manifest:
+    """Load and validate ``manifest.json`` from ``doc_dir``."""
+    path = doc_dir / MANIFEST_FILENAME
+    if not path.exists():
+        msg = f"manifest.json not found in {doc_dir}"
+        raise IndexNotFoundError(msg, details={"path": str(path)})
+
+    with path.open("r", encoding="utf-8") as fh:
+        payload = json.load(fh)
+
+    version = payload.get("format_version")
+    if version != MANIFEST_FORMAT_VERSION:
+        msg = (
+            f"unsupported manifest format version: {version!r} "
+            f"(expected {MANIFEST_FORMAT_VERSION})"
+        )
+        raise IndexNotFoundError(msg, details={"path": str(path)})
+
+    return Manifest.model_validate(payload)

cairn/entity/__init__.py

@@ -0,0 +1,21 @@
+"""Entity extraction — pluggable extractors that mine entities from a Document.
+
+Used by ``cairn.index.entities.EntityBuilder`` at indexing time. The
+heuristic extractor is the v0.2.0 default; an LLM-backed extractor for
+``term`` and ``proper`` kinds is planned for v0.2.1.
+
+Per ARCHITECTURE.md §2.3, entities come in four kinds — ``term``, ``code``,
+``proper``, ``defined``. The heuristic extractor covers ``code`` and
+``defined`` without any model dependency.
+"""
+
+from cairn.entity.base import EntityExtractor, ExtractionHit
+from cairn.entity.fake import FakeEntityExtractor
+from cairn.entity.heuristic import HeuristicExtractor
+
+__all__ = [
+    "EntityExtractor",
+    "ExtractionHit",
+    "FakeEntityExtractor",
+    "HeuristicExtractor",
+]

cairn/entity/base.py

@@ -0,0 +1,52 @@
+"""EntityExtractor protocol + intermediate extraction hit type.
+
+Extractors emit a sequence of :class:`ExtractionHit` — one per *occurrence*.
+The :class:`cairn.index.entities.EntityBuilder` deduplicates hits by
+``(canonical, kind)`` into :class:`cairn.core.types.Entity` records.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Iterable
+from typing import Protocol, runtime_checkable
+
+from pydantic import BaseModel, ConfigDict
+
+from cairn.core.types import Document, EntityKind, Span
+
+
+class ExtractionHit(BaseModel):
+    """One observed occurrence of a candidate entity.
+
+    Spans are offsets *within the section's ``raw_text``*, not into the
+    source document. The Entities sub-index stores spans in the same
+    coordinate space, so consumers do not need to know about section
+    territory boundaries to interpret them.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    section_id: str
+    canonical: str
+    surface_form: str
+    kind: EntityKind
+    span: Span
+
+
+@runtime_checkable
+class EntityExtractor(Protocol):
+    """A pluggable extractor.
+
+    Implementations may be sync (heuristic, regex-based) or async (LLM-backed).
+    The protocol uses an async signature; sync implementations return an
+    already-resolved awaitable.
+    """
+
+    name: str
+
+    def extract(
+        self,
+        document: Document,
+    ) -> Awaitable[Iterable[ExtractionHit]]:
+        """Return an iterable of extraction hits across ``document``."""
+        ...

cairn/entity/fake.py

@@ -0,0 +1,34 @@
+"""Deterministic entity extractor for tests.
+
+Returns a fixed catalogue of hits regardless of input. Used by tests that
+care about the downstream builder/index/tool behavior, not the extraction
+heuristics themselves.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from cairn.core.types import Document, Span
+from cairn.entity.base import ExtractionHit
+
+
+class FakeEntityExtractor:
+    """Returns one hit per section, kind=defined, canonical=<section_id>."""
+
+    name = "fake:per-section"
+
+    async def extract(self, document: Document) -> Iterable[ExtractionHit]:
+        hits: list[ExtractionHit] = []
+        for section in document.sections:
+            canonical = section.id.split("/")[-1].replace("-", " ")
+            hits.append(
+                ExtractionHit(
+                    section_id=section.id,
+                    canonical=canonical,
+                    surface_form=canonical,
+                    kind="defined",
+                    span=Span(start=0, end=min(len(canonical), len(section.raw_text))),
+                )
+            )
+        return hits