docsgraph 0.1.0a2__py3-none-any.whl

cairn/index/xrefs.py

@@ -0,0 +1,195 @@
+"""XRefs sub-index — cross-references (X), the document graph.
+
+The :class:`XRefBuilder` runs an :class:`cairn.xref.base.XRefExtractor`
+over a Document (plus an optional Entities reader), deduplicates
+``(src, dst, kind)`` triples by keeping the highest-confidence span, and
+writes ``refs.json``.
+
+The :class:`XRefs` reader exposes outgoing/incoming/by-kind queries. Edges
+are directed; a backward edge between two sections is its own record.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from collections.abc import Iterable, Iterator
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Final
+
+from cairn.core.errors import IndexBuildError, IndexNotFoundError
+from cairn.core.types import Document, Span, XRef, XRefKind
+from cairn.index.entities import Entities
+from cairn.xref.base import ExtractionEdge, XRefExtractor
+
+XREFS_FILENAME: Final = "refs.json"
+XREFS_FORMAT_VERSION: Final = 1
+
+
+class XRefBuilder:
+    """Run an extractor, aggregate edges, persist ``refs.json``."""
+
+    def __init__(self, extractor: XRefExtractor) -> None:
+        self.extractor = extractor
+
+    async def build(
+        self,
+        document: Document,
+        *,
+        out_dir: Path,
+        entities: Entities | None = None,
+    ) -> Path:
+        out_dir.mkdir(parents=True, exist_ok=True)
+        path = out_dir / XREFS_FILENAME
+
+        edges = await self.extractor.extract(document, entities=entities)
+        refs = _aggregate(edges)
+        now = datetime.now(UTC)
+
+        payload: dict[str, Any] = {
+            "format_version": XREFS_FORMAT_VERSION,
+            "doc_id": document.id,
+            "extractor": self.extractor.name,
+            "generated_at": now.isoformat(),
+            "refs": [_xref_to_dict(r) for r in refs],
+        }
+
+        with path.open("w", encoding="utf-8") as fh:
+            json.dump(payload, fh, ensure_ascii=False, indent=2)
+            fh.write("\n")
+        return path
+
+
+class XRefs:
+    """Loaded cross-references sub-index. Read-only queries."""
+
+    def __init__(
+        self,
+        refs: tuple[XRef, ...],
+        *,
+        doc_id: str,
+        extractor: str,
+    ) -> None:
+        self._all = refs
+        self.doc_id = doc_id
+        self.extractor = extractor
+
+        self._outgoing: dict[str, list[XRef]] = defaultdict(list)
+        self._incoming: dict[str, list[XRef]] = defaultdict(list)
+        for ref in refs:
+            self._outgoing[ref.src].append(ref)
+            self._incoming[ref.dst].append(ref)
+
+    @classmethod
+    def load(cls, doc_dir: Path) -> XRefs:
+        path = doc_dir / XREFS_FILENAME
+        if not path.exists():
+            msg = f"refs.json not found in {doc_dir}"
+            raise IndexNotFoundError(msg, details={"path": str(path)})
+
+        with path.open("r", encoding="utf-8") as fh:
+            payload = json.load(fh)
+
+        version = payload.get("format_version")
+        if version != XREFS_FORMAT_VERSION:
+            msg = (
+                f"unsupported refs format version: {version!r} "
+                f"(expected {XREFS_FORMAT_VERSION})"
+            )
+            raise IndexNotFoundError(msg, details={"path": str(path)})
+
+        refs = tuple(_xref_from_dict(d) for d in payload["refs"])
+        return cls(refs, doc_id=payload["doc_id"], extractor=payload["extractor"])
+
+    # -- queries -----------------------------------------------------------
+
+    def __len__(self) -> int:
+        return len(self._all)
+
+    def __iter__(self) -> Iterator[XRef]:
+        return iter(self._all)
+
+    def outgoing_from(
+        self, section_id: str, *, kinds: tuple[XRefKind, ...] | None = None
+    ) -> list[XRef]:
+        """Outgoing edges sorted by confidence descending."""
+        edges = self._outgoing.get(section_id, ())
+        if kinds is not None:
+            edges = [e for e in edges if e.kind in kinds]
+        return sorted(edges, key=lambda r: (-r.confidence, r.dst))
+
+    def incoming_to(
+        self, section_id: str, *, kinds: tuple[XRefKind, ...] | None = None
+    ) -> list[XRef]:
+        edges = self._incoming.get(section_id, ())
+        if kinds is not None:
+            edges = [e for e in edges if e.kind in kinds]
+        return sorted(edges, key=lambda r: (-r.confidence, r.src))
+
+    def by_kind(self, kind: XRefKind) -> list[XRef]:
+        return [r for r in self._all if r.kind == kind]
+
+
+# ---------------------------------------------------------------------------
+# Aggregation
+# ---------------------------------------------------------------------------
+
+
+def _aggregate(edges: Iterable[ExtractionEdge]) -> list[XRef]:
+    """Deduplicate ``(src, dst, kind)``; keep highest-confidence span."""
+    by_key: dict[tuple[str, str, str], XRef] = {}
+    insertion_order: list[tuple[str, str, str]] = []
+
+    for edge in edges:
+        if edge.src == edge.dst:
+            continue
+        if not edge.src or not edge.dst:
+            msg = "extractor emitted an edge with empty endpoint id"
+            raise IndexBuildError(msg)
+        key = (edge.src, edge.dst, edge.kind)
+        existing = by_key.get(key)
+        if existing is None:
+            by_key[key] = XRef(
+                src=edge.src,
+                dst=edge.dst,
+                kind=edge.kind,
+                confidence=edge.confidence,
+                span=edge.span,
+            )
+            insertion_order.append(key)
+        elif edge.confidence > existing.confidence:
+            by_key[key] = XRef(
+                src=edge.src,
+                dst=edge.dst,
+                kind=edge.kind,
+                confidence=edge.confidence,
+                span=edge.span,
+            )
+
+    return [by_key[key] for key in insertion_order]
+
+
+# ---------------------------------------------------------------------------
+# Serialization
+# ---------------------------------------------------------------------------
+
+
+def _xref_to_dict(r: XRef) -> dict[str, Any]:
+    return {
+        "src": r.src,
+        "dst": r.dst,
+        "kind": r.kind,
+        "confidence": r.confidence,
+        "span": {"start": r.span.start, "end": r.span.end},
+    }
+
+
+def _xref_from_dict(d: dict[str, Any]) -> XRef:
+    return XRef(
+        src=d["src"],
+        dst=d["dst"],
+        kind=d["kind"],
+        confidence=d["confidence"],
+        span=Span(start=d["span"]["start"], end=d["span"]["end"]),
+    )

cairn/ingest/__init__.py

@@ -0,0 +1,36 @@
+"""Ingestion layer — parsers from source formats into the canonical Document AST."""
+
+from cairn.ingest.base import Parser
+from cairn.ingest.markdown import MarkdownParser
+from cairn.ingest.markitdown import MarkItDownParser
+from cairn.ingest.pdf import PdfParser
+
+__all__ = ["MarkItDownParser", "MarkdownParser", "Parser", "PdfParser"]
+
+
+def parser_for_path(path) -> Parser:  # type: ignore[no-untyped-def]
+    """Pick a parser based on the file's extension.
+
+    Raises :class:`cairn.core.errors.ConfigError` for unknown extensions.
+    """
+    from pathlib import Path
+
+    from cairn.core.errors import ConfigError
+
+    p = Path(path)
+    ext = p.suffix.lower()
+    if ext in MarkdownParser.extensions:
+        return MarkdownParser()
+    if ext in PdfParser.extensions:
+        return PdfParser()
+    if ext in MarkItDownParser.extensions:
+        return MarkItDownParser()
+    msg = f"no parser registered for extension {ext!r}"
+    raise ConfigError(msg, details={"path": str(p), "extension": ext})
+
+
+def supported_extensions() -> frozenset[str]:
+    """Return every file extension Cairn can dispatch to an ingest parser."""
+    return frozenset(
+        (*MarkdownParser.extensions, *PdfParser.extensions, *MarkItDownParser.extensions)
+    )

cairn/ingest/base.py

@@ -0,0 +1,46 @@
+"""Parser protocol — the contract every ingestion plugin must satisfy."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from cairn.core.types import Document
+
+
+@runtime_checkable
+class Parser(Protocol):
+    """A source-format parser.
+
+    Implementations live in `cairn.plugins.*` or `cairn.ingest.*`. They must
+    preserve heading hierarchy, emit stable slug-based section IDs, and
+    populate byte spans into the original source.
+
+    See ARCHITECTURE.md §2.1 for the full set of hard rules.
+    """
+
+    name: str
+    """Identifier used in config (e.g. ``markdown``, ``pdf``)."""
+
+    extensions: tuple[str, ...]
+    """File extensions this parser claims, with leading dot. e.g. ``(".md",)``."""
+
+    def parse(
+        self,
+        source: Path | bytes | str,
+        *,
+        doc_id: str | None = None,
+    ) -> Document:
+        """Parse ``source`` into a canonical :class:`Document`.
+
+        Args:
+            source: A path, raw bytes, or text.
+            doc_id: Optional explicit document identifier. Required when
+                ``source`` is bytes or text. When a path is given and
+                ``doc_id`` is omitted, the parser derives it from the
+                filename stem.
+
+        Returns:
+            A fully populated :class:`Document` with section tree and spans.
+        """
+        ...

cairn/ingest/markdown.py

@@ -0,0 +1,244 @@
+"""Markdown parser — Markdown source → canonical Document AST.
+
+Preserves heading hierarchy, generates stable hierarchical slug-based section
+IDs, computes byte spans, and emits ``raw_text`` that excludes descendant
+section bodies (per ARCHITECTURE.md §2.2).
+
+Front-matter (YAML, TOML) is parsed and discarded. Content preceding the first
+heading is discarded; if a document has no headings, an empty section list is
+returned (callers may choose to treat this as a parse error).
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections import defaultdict
+from datetime import UTC, datetime
+from pathlib import Path
+
+from markdown_it import MarkdownIt
+from markdown_it.token import Token
+from mdit_py_plugins.front_matter import front_matter_plugin
+from slugify import slugify
+
+from cairn import __version__
+from cairn.core.errors import ParseError
+from cairn.core.types import Document, SectionNode, Span
+
+
+class MarkdownParser:
+    """CommonMark-compliant Markdown parser with front-matter and tables."""
+
+    name = "markdown"
+    extensions: tuple[str, ...] = (".md", ".markdown", ".mdown", ".mkd")
+
+    def __init__(self) -> None:
+        md = MarkdownIt("commonmark", {"html": False})
+        md.use(front_matter_plugin)
+        md.enable(["table"])
+        self._md = md
+
+    def parse(
+        self,
+        source: Path | bytes | str,
+        *,
+        doc_id: str | None = None,
+    ) -> Document:
+        source_path, text, derived_doc_id = self._resolve_source(source, doc_id)
+
+        text_bytes = text.encode("utf-8")
+        line_offsets = _compute_line_offsets(text_bytes)
+        tokens = self._md.parse(text)
+        headings = _extract_headings(tokens)
+        sections = _build_sections(headings, text_bytes, line_offsets)
+
+        return Document(
+            id=derived_doc_id,
+            source_path=source_path,
+            source_hash=hashlib.sha256(text_bytes).hexdigest(),
+            sections=tuple(sections),
+            indexed_at=datetime.now(UTC),
+            cairn_version=__version__,
+        )
+
+    @staticmethod
+    def _resolve_source(
+        source: Path | bytes | str,
+        doc_id: str | None,
+    ) -> tuple[Path, str, str]:
+        if isinstance(source, Path):
+            try:
+                text = source.read_text(encoding="utf-8")
+            except OSError as exc:
+                msg = f"could not read source file: {source}"
+                raise ParseError(msg, details={"path": str(source)}) from exc
+            resolved_id = doc_id or _slug_or_raise(source.stem, ctx="filename stem")
+            return source, text, resolved_id
+
+        if doc_id is None:
+            msg = "doc_id is required when source is not a path"
+            raise ParseError(msg)
+
+        text = source.decode("utf-8") if isinstance(source, bytes) else source
+        return Path(f"<in-memory:{doc_id}>"), text, doc_id
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _compute_line_offsets(text_bytes: bytes) -> list[int]:
+    """Return byte offsets where each line begins.
+
+    The list has length ``num_lines + 1``: the final element equals
+    ``len(text_bytes)``, acting as a virtual one-past-the-end line start so
+    callers can address EOF uniformly.
+    """
+    offsets = [0]
+    for i, byte in enumerate(text_bytes):
+        if byte == 0x0A:  # newline
+            offsets.append(i + 1)
+    offsets.append(len(text_bytes))
+    return offsets
+
+
+def _heading_title(inline_tok: Token) -> str:
+    """Extract plain text title from a heading's inline token.
+
+    Concatenates the text content of leaf ``text`` children, dropping markup.
+    Falls back to the raw inline content if no children are present.
+    """
+    if inline_tok.children is None:
+        return inline_tok.content
+    parts: list[str] = []
+    for child in inline_tok.children:
+        if child.type in ("text", "code_inline"):
+            parts.append(child.content)
+    return "".join(parts).strip() or inline_tok.content.strip()
+
+
+def _extract_headings(tokens: list[Token]) -> list[tuple[int, str, int, int]]:
+    """Return ``(level, title, line_start, line_end_excl)`` for each heading."""
+    headings: list[tuple[int, str, int, int]] = []
+    for i, tok in enumerate(tokens):
+        if tok.type != "heading_open" or tok.map is None:
+            continue
+        level = int(tok.tag[1])
+        line_start, line_end_excl = tok.map
+        # Inline token always follows heading_open.
+        if i + 1 >= len(tokens) or tokens[i + 1].type != "inline":
+            continue
+        title = _heading_title(tokens[i + 1])
+        headings.append((level, title, line_start, line_end_excl))
+    return headings
+
+
+def _slug_or_raise(text: str, *, ctx: str) -> str:
+    slug = slugify(text)
+    if not slug:
+        msg = f"could not derive a slug from {ctx}: {text!r}"
+        raise ParseError(msg)
+    return slug
+
+
+def _safe_slug(text: str) -> str:
+    """Slug that always returns something usable; falls back to ``section``."""
+    return slugify(text) or "section"
+
+
+def _build_sections(
+    headings: list[tuple[int, str, int, int]],
+    text_bytes: bytes,
+    line_offsets: list[int],
+) -> list[SectionNode]:
+    """Assemble SectionNode objects from heading metadata."""
+    if not headings:
+        return []
+
+    n = len(headings)
+    total_bytes = len(text_bytes)
+
+    # Territory: end at next heading with level <= current_level (else EOF).
+    territory_end_line: list[int] = []
+    for i, (level, _, _, _) in enumerate(headings):
+        next_terr = len(line_offsets) - 1
+        for j in range(i + 1, n):
+            if headings[j][0] <= level:
+                next_terr = headings[j][2]
+                break
+        territory_end_line.append(next_terr)
+
+    # Raw text end: next heading at ANY level (else territory end).
+    raw_text_end_line: list[int] = []
+    for i in range(n):
+        if i + 1 < n:
+            raw_text_end_line.append(headings[i + 1][2])
+        else:
+            raw_text_end_line.append(territory_end_line[i])
+
+    # Hierarchical IDs, parents, paths.
+    metadata: list[tuple[str, str, int, str | None, tuple[str, ...]]] = []
+    stack: list[tuple[int, str, str]] = []  # (level, id, title)
+    sibling_counters: dict[tuple[str, str], int] = defaultdict(int)
+
+    for level, title, _line_start, _line_end in headings:
+        while stack and stack[-1][0] >= level:
+            stack.pop()
+
+        parent_id = stack[-1][1] if stack else None
+        slug = _safe_slug(title)
+        key = (parent_id or "", slug)
+        sibling_counters[key] += 1
+        count = sibling_counters[key]
+        unique_slug = slug if count == 1 else f"{slug}-{count}"
+
+        section_id = f"{parent_id}/{unique_slug}" if parent_id else unique_slug
+        path = (*(t for _, _, t in stack), title)
+
+        metadata.append((section_id, title, level, parent_id, path))
+        stack.append((level, section_id, title))
+
+    children_map: dict[str, list[str]] = defaultdict(list)
+    for sid, _title, _level, parent_id, _path in metadata:
+        if parent_id is not None:
+            children_map[parent_id].append(sid)
+
+    sections: list[SectionNode] = []
+    for idx, (level, _title_h, _line_start, line_end_excl) in enumerate(headings):
+        section_id, title, _lvl, parent_id, path = metadata[idx]
+
+        span_start_line = headings[idx][2]
+        span_end_line = territory_end_line[idx]
+        span = Span(
+            start=_line_to_byte(line_offsets, span_start_line, total_bytes),
+            end=_line_to_byte(line_offsets, span_end_line, total_bytes),
+        )
+
+        raw_start = _line_to_byte(line_offsets, line_end_excl, total_bytes)
+        raw_end = _line_to_byte(line_offsets, raw_text_end_line[idx], total_bytes)
+        raw_text = text_bytes[raw_start:raw_end].decode("utf-8")
+
+        sections.append(
+            SectionNode(
+                id=section_id,
+                title=title,
+                level=level,
+                parent=parent_id,
+                children=tuple(children_map.get(section_id, ())),
+                span=span,
+                path=path,
+                raw_text=raw_text,
+            )
+        )
+
+    return sections
+
+
+def _line_to_byte(line_offsets: list[int], line: int, total_bytes: int) -> int:
+    """Resolve a 0-indexed line number to its starting byte offset."""
+    if line >= len(line_offsets):
+        return total_bytes
+    if line < 0:
+        return 0
+    return line_offsets[line]

cairn/ingest/markitdown.py

@@ -0,0 +1,145 @@
+"""Optional MarkItDown-backed parser for office/data/web document formats.
+
+MarkItDown converts many source formats into Markdown intended for LLM and
+text-analysis pipelines. Cairn uses it as a local-file conversion layer, then
+delegates structure extraction to :class:`MarkdownParser`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from typing import Any
+
+from slugify import slugify
+
+from cairn.core.errors import ParseError
+from cairn.core.types import Document
+from cairn.ingest.markdown import MarkdownParser
+
+
+class MarkItDownParser:
+    """Convert non-native local files to Markdown and parse the result."""
+
+    name = "markitdown"
+    extensions: tuple[str, ...] = (
+        ".docx",
+        ".pptx",
+        ".xlsx",
+        ".xls",
+        ".html",
+        ".htm",
+        ".csv",
+        ".json",
+        ".xml",
+        ".epub",
+        ".msg",
+        ".png",
+        ".jpg",
+        ".jpeg",
+        ".gif",
+        ".wav",
+        ".mp3",
+        ".zip",
+    )
+
+    def __init__(self) -> None:
+        self._markdown = MarkdownParser()
+
+    def parse(
+        self,
+        source: Path | bytes | str,
+        *,
+        doc_id: str | None = None,
+    ) -> Document:
+        if not isinstance(source, Path):
+            msg = "MarkItDownParser only accepts local file paths"
+            raise ParseError(msg)
+
+        source_path = source.resolve()
+        resolved_doc_id = doc_id or _slug_or_raise(source_path.stem, ctx="filename stem")
+        markdown = self._convert_local(source_path)
+        if not markdown.strip():
+            msg = f"MarkItDown produced empty Markdown for: {source_path}"
+            raise ParseError(msg, details={"path": str(source_path)})
+        markdown = _ensure_heading(markdown, source_path.stem)
+
+        parsed = self._markdown.parse(markdown, doc_id=resolved_doc_id)
+        try:
+            source_bytes = source_path.read_bytes()
+        except OSError as exc:
+            msg = f"could not read converted source file: {source_path}"
+            raise ParseError(msg, details={"path": str(source_path)}) from exc
+        converter_version = _markitdown_version()
+        source_hash = hashlib.sha256(
+            b"\x00".join(
+                [
+                    source_bytes,
+                    converter_version.encode("utf-8"),
+                    markdown.encode("utf-8"),
+                ]
+            )
+        ).hexdigest()
+        return parsed.model_copy(
+            update={
+                "source_path": source_path,
+                "source_hash": source_hash,
+            }
+        )
+
+    @staticmethod
+    def _convert_local(path: Path) -> str:
+        try:
+            from markitdown import MarkItDown
+        except ImportError as exc:
+            msg = (
+                "MarkItDown support is not installed. Install with "
+                "`pip install 'cairn[markitdown]'` or "
+                "`uv pip install -e '.[markitdown]'`."
+            )
+            raise ParseError(msg, details={"path": str(path)}) from exc
+
+        try:
+            converter: Any = MarkItDown(enable_plugins=False)
+            if hasattr(converter, "convert_local"):
+                result = converter.convert_local(str(path))
+            else:
+                result = converter.convert(str(path))
+        except Exception as exc:
+            msg = f"MarkItDown could not convert local file: {path}"
+            raise ParseError(msg, details={"path": str(path)}) from exc
+
+        text = getattr(result, "text_content", None)
+        if text is None:
+            text = getattr(result, "markdown", None)
+        if not isinstance(text, str):
+            msg = "MarkItDown returned no Markdown text content"
+            raise ParseError(msg, details={"path": str(path)})
+        return text
+
+
+def _slug_or_raise(text: str, *, ctx: str) -> str:
+    slug = slugify(text)
+    if not slug:
+        msg = f"could not derive a slug from {ctx}: {text!r}"
+        raise ParseError(msg)
+    return slug
+
+
+def _markitdown_version() -> str:
+    try:
+        return version("markitdown")
+    except PackageNotFoundError:
+        return "not-installed"
+
+
+def _ensure_heading(markdown: str, fallback_title: str) -> str:
+    for line in markdown.splitlines():
+        stripped = line.strip()
+        if stripped.startswith("# "):
+            return markdown
+        if stripped:
+            break
+    title = fallback_title.replace("_", " ").replace("-", " ").strip() or "Document"
+    return f"# {title}\n\n{markdown}"