docsgraph 0.1.0a2__py3-none-any.whl

cairn/cli/config.py

@@ -0,0 +1,105 @@
+"""CLI configuration — environment variables with Ollama defaults.
+
+Per CLAUDE.md P4 ("local-first must always work"), the defaults target a
+local Ollama instance and require no API key.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Literal, cast
+
+from pydantic import BaseModel, ConfigDict
+
+EmbedProvider = Literal["openai-compatible", "doubao-vision"]
+
+
+class LLMConfig(BaseModel):
+    """Summarizer endpoint configuration."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    base_url: str = "http://localhost:11434/v1"
+    model: str = "llama3.2:3b"
+    api_key: str | None = None
+    timeout: float = 60.0
+    max_retries: int = 2
+
+
+class EmbedConfig(BaseModel):
+    """Embedder endpoint configuration."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    provider: EmbedProvider = "openai-compatible"
+    base_url: str = "http://localhost:11434/v1"
+    model: str = "nomic-embed-text"
+    dim: int = 768
+    api_key: str | None = None
+    timeout: float = 60.0
+    max_retries: int = 2
+
+
+class IndexConfig(BaseModel):
+    """Index-build performance knobs."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    summary_concurrency: int = 4
+    embed_batch_size: int = 32
+
+
+def load_llm_config() -> LLMConfig:
+    """Read summarizer config from ``CAIRN_LLM_*`` environment variables."""
+    return LLMConfig(
+        base_url=os.environ.get("CAIRN_LLM_BASE_URL", "http://localhost:11434/v1"),
+        model=os.environ.get("CAIRN_LLM_MODEL", "llama3.2:3b"),
+        api_key=os.environ.get("CAIRN_LLM_API_KEY") or None,
+        timeout=_float_env("CAIRN_LLM_TIMEOUT", 60.0),
+        max_retries=_int_env("CAIRN_LLM_MAX_RETRIES", 2),
+    )
+
+
+def load_embed_config() -> EmbedConfig:
+    """Read embedder config from ``CAIRN_EMBED_*`` environment variables."""
+    provider = os.environ.get("CAIRN_EMBED_PROVIDER", "openai-compatible")
+    if provider == "doubao-vision":
+        default_base_url = "https://ark.cn-beijing.volces.com/api/v3"
+        default_model = "doubao-embedding-vision-251215"
+        default_dim = "2048"
+    else:
+        default_base_url = "http://localhost:11434/v1"
+        default_model = "nomic-embed-text"
+        default_dim = "768"
+
+    return EmbedConfig(
+        provider=cast(EmbedProvider, provider),
+        base_url=os.environ.get("CAIRN_EMBED_BASE_URL", default_base_url),
+        model=os.environ.get("CAIRN_EMBED_MODEL", default_model),
+        dim=int(os.environ.get("CAIRN_EMBED_DIM", default_dim)),
+        api_key=os.environ.get("CAIRN_EMBED_API_KEY") or None,
+        timeout=_float_env("CAIRN_EMBED_TIMEOUT", 60.0),
+        max_retries=_int_env("CAIRN_EMBED_MAX_RETRIES", 2),
+    )
+
+
+def load_index_config() -> IndexConfig:
+    """Read index-build performance config from environment variables."""
+    return IndexConfig(
+        summary_concurrency=_int_env("CAIRN_SUMMARY_CONCURRENCY", 4),
+        embed_batch_size=_int_env("CAIRN_EMBED_BATCH_SIZE", 32),
+    )
+
+
+def _int_env(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    if raw is None or not raw.strip():
+        return default
+    return int(raw)
+
+
+def _float_env(name: str, default: float) -> float:
+    raw = os.environ.get(name)
+    if raw is None or not raw.strip():
+        return default
+    return float(raw)

cairn/core/__init__.py

@@ -0,0 +1,41 @@
+"""Core types, errors, and configuration for Cairn."""
+
+from cairn.core.errors import (
+    CairnError,
+    ConfigError,
+    IndexBuildError,
+    IndexNotFoundError,
+    IndexStaleError,
+    ParseError,
+    ToolError,
+)
+from cairn.core.types import (
+    Document,
+    Entity,
+    EntityKind,
+    Mention,
+    SectionNode,
+    Span,
+    SummarySet,
+    XRef,
+    XRefKind,
+)
+
+__all__ = [
+    "CairnError",
+    "ConfigError",
+    "Document",
+    "Entity",
+    "EntityKind",
+    "IndexBuildError",
+    "IndexNotFoundError",
+    "IndexStaleError",
+    "Mention",
+    "ParseError",
+    "SectionNode",
+    "Span",
+    "SummarySet",
+    "ToolError",
+    "XRef",
+    "XRefKind",
+]

cairn/core/errors.py

@@ -0,0 +1,68 @@
+"""Cairn error hierarchy.
+
+Every error raised by Cairn library code derives from `CairnError`. Tool-layer
+code translates these into structured MCP error envelopes; never lets them
+escape to the transport.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class CairnError(Exception):
+    """Base class for all Cairn errors."""
+
+    code: str = "INTERNAL"
+
+    def __init__(self, message: str, *, details: dict[str, Any] | None = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.details: dict[str, Any] = details or {}
+
+    def to_envelope(self) -> dict[str, Any]:
+        """Convert to the structured MCP error payload.
+
+        See docs/specs/mcp-tools.md §0 for the envelope shape.
+        """
+        return {
+            "code": self.code,
+            "message": self.message,
+            "details": self.details,
+        }
+
+
+class ParseError(CairnError):
+    """Source document could not be parsed into a canonical Document AST."""
+
+    code = "PARSE_FAILED"
+
+
+class IndexBuildError(CairnError):
+    """An index builder failed while constructing or updating an artifact."""
+
+    code = "INDEX_BUILD_FAILED"
+
+
+class IndexNotFoundError(CairnError):
+    """A referenced index or section does not exist."""
+
+    code = "NOT_FOUND"
+
+
+class IndexStaleError(CairnError):
+    """The on-disk index is older than its source and must be rebuilt."""
+
+    code = "INDEX_STALE"
+
+
+class ConfigError(CairnError):
+    """Invalid or missing configuration."""
+
+    code = "INVALID_CONFIG"
+
+
+class ToolError(CairnError):
+    """An MCP tool received invalid input or could not produce a result."""
+
+    code = "INVALID_INPUT"

cairn/core/types.py

@@ -0,0 +1,147 @@
+"""Canonical Cairn data model.
+
+These are the types that flow across layer boundaries. They are the contract
+between ingestion, indexing, retrieval, and the MCP server. Treat them as the
+schema; never substitute ad-hoc dicts.
+
+Mirrors ARCHITECTURE.md §4. Changes here are breaking and require an ADR.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
+
+EntityKind = Literal["term", "code", "proper", "defined"]
+XRefKind = Literal["link", "textual", "entity"]
+
+
+class _Frozen(BaseModel):
+    """Common config for immutable, strict models."""
+
+    model_config = ConfigDict(
+        frozen=True,
+        extra="forbid",
+        str_strip_whitespace=False,
+        validate_assignment=True,
+    )
+
+
+class Span(_Frozen):
+    """A half-open byte range `[start, end)` in the source document."""
+
+    start: int = Field(ge=0)
+    end: int = Field(ge=0)
+
+    @field_validator("end")
+    @classmethod
+    def _end_after_start(cls, end: int, info: ValidationInfo) -> int:
+        start = info.data.get("start")
+        if isinstance(start, int) and end < start:
+            msg = f"Span.end ({end}) must be >= Span.start ({start})"
+            raise ValueError(msg)
+        return end
+
+    def __len__(self) -> int:
+        return self.end - self.start
+
+
+class SectionNode(_Frozen):
+    """A node in the document's structural tree.
+
+    `id` is hierarchical, slug-based, and stable across re-indexing of the
+    same document. Example: ``hooks/use-effect/cleanup``.
+
+    `raw_text` is the body that belongs **directly** to this section, excluding
+    any descendant sections' bodies. To read continuous text including
+    descendants, use the `span` and read from the source — or use the
+    `read_range` retrieval tool.
+    """
+
+    id: str = Field(min_length=1)
+    title: str
+    level: int = Field(ge=1, le=6)
+    parent: str | None
+    children: tuple[str, ...] = ()
+    span: Span
+    path: tuple[str, ...]
+    raw_text: str
+
+    @field_validator("id")
+    @classmethod
+    def _id_well_formed(cls, value: str) -> str:
+        if value.startswith("/") or value.endswith("/"):
+            msg = f"section id must not start or end with '/': {value!r}"
+            raise ValueError(msg)
+        if "//" in value:
+            msg = f"section id must not contain '//': {value!r}"
+            raise ValueError(msg)
+        return value
+
+
+class SummarySet(_Frozen):
+    """Multi-granularity summaries for one section.
+
+    Generated by a `Summarizer` during indexing. Never produced at query time.
+
+    `digest` is optional because v0.1 generates only gist + synopsis; the
+    deeper level lands in v0.2. A `None` digest means "not generated at the
+    time this summary was built", not "this section has no digest possible".
+    """
+
+    section_id: str
+    gist: str = Field(description="≤ 20 words; the 'scent' in IFT terms")
+    synopsis: str = Field(description="one paragraph; ≤ 80 words")
+    digest: str | None = Field(
+        default=None,
+        description="multi-paragraph; ≤ 300 words. None until v0.2.",
+    )
+    model: str = Field(description="identifier of the LLM that produced these")
+    section_hash: str = Field(
+        description="sha256 hex of (title + raw_text) at generation time; "
+        "used to detect stale summaries when raw_text changes."
+    )
+    generated_at: datetime
+
+
+class Mention(_Frozen):
+    """A single occurrence of an entity inside a section."""
+
+    section_id: str
+    span: Span
+
+
+class Entity(_Frozen):
+    """A canonicalized term/concept and its mentions throughout the document."""
+
+    canonical: str = Field(min_length=1)
+    surface_forms: tuple[str, ...]
+    kind: EntityKind
+    mentions: tuple[Mention, ...]
+
+
+class XRef(_Frozen):
+    """A directed edge in the cross-reference graph."""
+
+    src: str = Field(description="section_id of the source")
+    dst: str = Field(description="section_id of the destination")
+    kind: XRefKind
+    confidence: float = Field(ge=0.0, le=1.0)
+    span: Span
+
+
+class Document(_Frozen):
+    """A fully ingested document.
+
+    Produced by Layer 1 (Ingestion). The sole input to Layer 2 (Index).
+    """
+
+    id: str = Field(min_length=1, description="human-readable doc_id, slug-based")
+    source_path: Path
+    source_hash: str = Field(description="sha256 of the source bytes, hex-encoded")
+    sections: tuple[SectionNode, ...]
+    indexed_at: datetime
+    cairn_version: str

cairn/embed/__init__.py

@@ -0,0 +1,17 @@
+"""Embedding layer — pluggable text → vector encoders.
+
+Used by the index layer (`cairn.index.vectors.VectorBuilder`) at indexing time.
+Never invoked at query time except for embedding the user's query string.
+"""
+
+from cairn.embed.base import Embedder
+from cairn.embed.doubao import DoubaoVisionEmbedder
+from cairn.embed.fake import FakeEmbedder
+from cairn.embed.openai_compatible import OpenAICompatibleEmbedder
+
+__all__ = [
+    "DoubaoVisionEmbedder",
+    "Embedder",
+    "FakeEmbedder",
+    "OpenAICompatibleEmbedder",
+]

cairn/embed/base.py

@@ -0,0 +1,31 @@
+"""Embedder protocol.
+
+An ``Embedder`` turns a list of texts into a list of dense vectors. Batching
+is the responsibility of the implementation — callers pass a list and may
+assume the implementation chooses an efficient batching strategy.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Embedder(Protocol):
+    """A pluggable text-embedding model.
+
+    The ``name`` attribute encodes both the implementation family and the
+    model identifier (e.g. ``"openai-compat:nomic-embed-text"``) so that
+    consumers can use it as a cache-invalidation key and a manifest marker.
+
+    Vectors returned by ``embed`` MUST have length ``dim`` for every text;
+    consumers may rely on this invariant when constructing typed vector
+    stores. Empty input must return an empty list (not raise).
+    """
+
+    name: str
+    dim: int
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        """Embed each text in ``texts`` to a ``dim``-dimensional vector."""
+        ...

cairn/embed/doubao.py

@@ -0,0 +1,167 @@
+"""Volcengine/Doubao embedding adapters."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import httpx
+
+from cairn.core.errors import IndexBuildError
+
+
+class DoubaoVisionEmbedder:
+    """Client for Doubao's multimodal embedding endpoint.
+
+    ``doubao-embedding-vision-*`` models do not use the OpenAI-compatible
+    ``/embeddings`` wire shape. They are served at
+    ``/embeddings/multimodal`` and return ``{"data": {"embedding": ...}}``
+    for a single multimodal input. Cairn's embedder protocol expects one
+    vector per text, so this adapter issues one request per input text.
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: str = "https://ark.cn-beijing.volces.com/api/v3",
+        model: str = "doubao-embedding-vision-251215",
+        dim: int = 2048,
+        api_key: str | None = None,
+        timeout: float = 60.0,
+        max_retries: int = 2,
+        retry_base_delay: float = 0.5,
+    ) -> None:
+        if dim < 1:
+            msg = f"dim must be >= 1; got {dim}"
+            raise ValueError(msg)
+        if max_retries < 0:
+            msg = f"max_retries must be >= 0; got {max_retries}"
+            raise ValueError(msg)
+        if retry_base_delay < 0:
+            msg = f"retry_base_delay must be >= 0; got {retry_base_delay}"
+            raise ValueError(msg)
+        self.base_url = base_url.rstrip("/")
+        self.model = model
+        self.dim = dim
+        self.api_key = api_key
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.retry_base_delay = retry_base_delay
+        self.name = f"doubao-vision:{model}"
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        """Embed each text through Doubao's multimodal vectorization API."""
+        if not texts:
+            return []
+
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        vectors: list[list[float]] = []
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            for index, text in enumerate(texts):
+                payload: dict[str, Any] = {
+                    "model": self.model,
+                    "input": [{"type": "text", "text": text}],
+                }
+                response = await self._post_with_retries(
+                    client, payload, headers, index=index
+                )
+
+                vector = _extract_vector(response.json(), index=index)
+                if len(vector) != self.dim:
+                    msg = (
+                        f"doubao vision embedder returned dim={len(vector)} "
+                        f"but client expects dim={self.dim} "
+                        f"(model {self.model!r}, index {index})"
+                    )
+                    raise IndexBuildError(msg)
+                vectors.append(vector)
+
+        return vectors
+
+    async def _post_with_retries(
+        self,
+        client: httpx.AsyncClient,
+        payload: dict[str, Any],
+        headers: dict[str, str],
+        *,
+        index: int,
+    ) -> httpx.Response:
+        last_exc: httpx.HTTPError | None = None
+        url = f"{self.base_url}/embeddings/multimodal"
+        for attempt in range(self.max_retries + 1):
+            try:
+                response = await client.post(url, json=payload, headers=headers)
+            except httpx.HTTPError as exc:
+                last_exc = exc
+                if attempt < self.max_retries:
+                    await self._sleep_before_retry(attempt)
+                    continue
+                msg = f"doubao vision embedder request failed: {exc}"
+                raise IndexBuildError(
+                    msg,
+                    details={
+                        "model": self.model,
+                        "base_url": self.base_url,
+                        "index": index,
+                        "error_type": type(exc).__name__,
+                        "attempts": attempt + 1,
+                    },
+                ) from exc
+
+            if response.status_code in (429, 500, 502, 503, 504) and attempt < self.max_retries:
+                await self._sleep_before_retry(attempt)
+                continue
+            if response.status_code >= 400:
+                msg = (
+                    f"doubao vision embedder endpoint returned HTTP "
+                    f"{response.status_code}: {response.text[:200]}"
+                )
+                raise IndexBuildError(
+                    msg,
+                    details={
+                        "status": response.status_code,
+                        "model": self.model,
+                        "base_url": self.base_url,
+                        "index": index,
+                        "attempts": attempt + 1,
+                    },
+                )
+            return response
+
+        msg = "doubao vision embedder request failed without a response"
+        raise IndexBuildError(
+            msg,
+            details={
+                "model": self.model,
+                "base_url": self.base_url,
+                "index": index,
+                "error_type": type(last_exc).__name__ if last_exc else None,
+            },
+        )
+
+    async def _sleep_before_retry(self, attempt: int) -> None:
+        if self.retry_base_delay == 0:
+            return
+        await asyncio.sleep(self.retry_base_delay * (2**attempt))
+
+
+def _extract_vector(data: dict[str, Any], *, index: int) -> list[float]:
+    """Read the dense vector from Doubao's multimodal response shape."""
+    try:
+        embedding = data["data"]["embedding"]
+    except (KeyError, TypeError) as exc:
+        msg = "doubao vision embedder response did not match expected shape"
+        raise IndexBuildError(msg, details={"response": data, "index": index}) from exc
+
+    if not isinstance(embedding, list):
+        msg = "doubao vision embedder embedding is not a list"
+        raise IndexBuildError(msg, details={"response": data, "index": index})
+
+    try:
+        return [float(value) for value in embedding]
+    except (TypeError, ValueError) as exc:
+        msg = "doubao vision embedder embedding contains non-numeric values"
+        raise IndexBuildError(msg, details={"response": data, "index": index}) from exc

cairn/embed/fake.py

@@ -0,0 +1,36 @@
+"""Deterministic, network-free embedder for tests and offline development.
+
+Implementation is a sparse bag-of-words hash projection: each word in the
+input lowers onto exactly one dimension (chosen by sha256 hash mod dim).
+Vectors are similarity-respecting — two texts that share words land near
+each other in cosine space — but the embedder has no semantic understanding.
+Suitable for unit tests and pipeline plumbing checks; never for production.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+
+class FakeEmbedder:
+    """Bag-of-words hash embedder. Deterministic; no network."""
+
+    name = "fake:bow-hash"
+
+    def __init__(self, dim: int = 64) -> None:
+        if dim < 1:
+            msg = f"dim must be >= 1; got {dim}"
+            raise ValueError(msg)
+        self.dim = dim
+
+    async def embed(self, texts: list[str]) -> list[list[float]]:
+        return [self._embed_one(t) for t in texts]
+
+    def _embed_one(self, text: str) -> list[float]:
+        vec = [0.0] * self.dim
+        words = text.lower().split() or ["__empty__"]
+        for word in words:
+            digest = hashlib.sha256(word.encode("utf-8")).digest()
+            idx = int.from_bytes(digest[:8], "big") % self.dim
+            vec[idx] += 1.0
+        return vec