tracecite 0.0.2__py3-none-any.whl

tracecite/docs/vectorize.py

@@ -0,0 +1,457 @@
+"""Adapter that feeds retained documentation into the existing evidence stack."""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from ..evidence.paths import file_sha256
+from ..tables import augment_document_with_embedding_text
+from .contract import DocsEvidenceContract
+from .stage import is_publication_excluded, validate_retained_source_links
+
+if TYPE_CHECKING:
+    from ..evidence.sync import SyncOptions, SyncReport
+
+INDEX_INPUT_NAME = "index-input"
+MANIFEST_NAME = "index-input.manifest.toml"
+
+_EXCLUDED_DIR_NAMES = {
+    ".git",
+    ".quarto",
+    "_quarto",
+    "_site",
+    "site_libs",
+    "_tracecite",
+    ".tracecite",
+    "build",
+    ".pytest_cache",
+    ".venv",
+    "dist",
+    "__pycache__",
+    "model-cache",
+}
+_EXCLUDED_SUFFIXES = {".html"}
+_RESOURCE_REFERENCE_RE = re.compile(
+    r"(?:!\[[^\]]*\]|\[[^\]]*\])\(([^)]+)\)|^\s*\[[^\]]+\]:\s*(\S+)",
+    re.MULTILINE,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class DocsIndexProfile:
+    input_root: Path
+    manifest_path: Path
+    database_path: Path
+    model_cache_dir: Path
+
+
+@dataclass(frozen=True, slots=True)
+class DocsIndexResult:
+    profile: DocsIndexProfile
+    tables_normalized: int
+    sync_report: SyncReport
+
+
+def resolve_docs_index_profile(contract: DocsEvidenceContract) -> DocsIndexProfile:
+    database_path = contract.index_output.resolve()
+    return DocsIndexProfile(
+        input_root=(contract.staged_root / INDEX_INPUT_NAME).resolve(),
+        manifest_path=(contract.staged_root / MANIFEST_NAME).resolve(),
+        database_path=database_path,
+        model_cache_dir=(database_path.parent / "model-cache").resolve(),
+    )
+
+
+def _inside(path: Path, root: Path) -> bool:
+    return path.resolve().is_relative_to(root.resolve())
+
+
+def _should_skip_retained_path(path: Path, retained_root: Path) -> bool:
+    relative = path.relative_to(retained_root)
+    return (
+        not path.is_file()
+        or any(
+            part in _EXCLUDED_DIR_NAMES or part.startswith(".tracecite")
+            for part in relative.parts
+        )
+        or path.suffix.lower() in _EXCLUDED_SUFFIXES
+        or path.name.endswith("_files")
+    )
+
+
+def _mirror_exclude_globs(contract: DocsEvidenceContract) -> tuple[str, ...]:
+    retained = contract.retained_root.resolve()
+    globs: set[str] = set()
+    for exclude in contract.publication_exclude:
+        target = exclude.resolve()
+        if not _inside(target, retained):
+            continue
+        relative = target.relative_to(retained).as_posix()
+        if not relative or relative == ".":
+            raise ValueError(
+                f"publication_exclude cannot exclude the entire retained root: {exclude}"
+            )
+        if ".." in relative.split("/"):
+            raise ValueError(
+                f"publication_exclude cannot be represented safely in the mirror: {exclude}"
+            )
+        globs.add(f"{relative}/**" if target.is_dir() else relative)
+    for name in sorted(_EXCLUDED_DIR_NAMES):
+        globs.add(f"**/{name}/**")
+    for suffix in sorted(_EXCLUDED_SUFFIXES):
+        globs.add(f"**/*{suffix}")
+    globs.add("**/*_files/**")
+    return tuple(sorted(globs))
+
+
+def _retained_markdown_paths(contract: DocsEvidenceContract) -> list[Path]:
+    retained = contract.retained_root.resolve()
+    return [
+        path
+        for path in sorted(retained.rglob("*.md"))
+        if not _should_skip_retained_path(path, retained)
+        and not is_publication_excluded(path, contract)
+    ]
+
+
+def _local_resource_destinations(markdown: str) -> set[str]:
+    destinations: set[str] = set()
+    for match in _RESOURCE_REFERENCE_RE.finditer(markdown):
+        destination = (match.group(1) or match.group(2) or "").strip()
+        if not destination or destination.startswith("<"):
+            continue
+        lowered = destination.lower()
+        if lowered.startswith(("http://", "https://", "mailto:")):
+            continue
+        if "#page=" in lowered or lowered.endswith(".pdf"):
+            continue
+        destination = destination.split("#", 1)[0].strip()
+        if destination:
+            destinations.add(destination)
+    return destinations
+
+
+def _copy_resource_files(
+    *,
+    source_markdown: Path,
+    retained_root: Path,
+    mirror_root: Path,
+    original_markdown: str,
+) -> None:
+    for destination in sorted(_local_resource_destinations(original_markdown)):
+        source = (source_markdown.parent / destination).resolve()
+        if not source.is_file() or not _inside(source, retained_root):
+            continue
+        target = mirror_root / source.relative_to(retained_root)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source, target)
+
+
+def _render_manifest(exclude_globs: tuple[str, ...]) -> str:
+    lines = ["schema_version = 1", "", "[[include]]", 'glob = "**/*.md"']
+    for pattern in exclude_globs:
+        lines.extend(["", "[[exclude]]", f'glob = "{pattern}"'])
+    return "\n".join(lines) + "\n"
+
+
+def _snapshot_retained(retained_root: Path) -> dict[Path, bytes]:
+    return {
+        path.relative_to(retained_root): path.read_bytes()
+        for path in sorted(retained_root.rglob("*"))
+        if path.is_file()
+    }
+
+
+def _assert_retained_unchanged(retained_root: Path, before: dict[Path, bytes]) -> None:
+    after = _snapshot_retained(retained_root)
+    if before != after:
+        changed = sorted(
+            set(before) ^ set(after)
+            | {path for path in before if before.get(path) != after.get(path)}
+        )
+        raise RuntimeError(
+            f"retained Markdown was mutated during index-input preparation: {changed[0]}"
+        )
+
+
+def _atomic_promote_pair(replacements: tuple[tuple[Path, Path], ...]) -> None:
+    """Replace every destination with its staged temp path, or roll all of them back.
+
+    ``replacements`` is a sequence of ``(temp_path, destination)`` pairs. If any
+    individual replace fails partway through, every destination already moved is
+    restored to its prior content and no stray ``.previous`` backup is left behind.
+    """
+    backups: list[tuple[Path, Path]] = []
+    try:
+        for _temp, destination in replacements:
+            if destination.exists():
+                backup = destination.with_name(f".{destination.name}.previous")
+                if backup.exists():
+                    shutil.rmtree(backup) if backup.is_dir() else backup.unlink()
+                os.replace(destination, backup)
+                backups.append((destination, backup))
+        for temp, destination in replacements:
+            os.replace(temp, destination)
+    except Exception:
+        for _temp, destination in replacements:
+            if destination.exists():
+                shutil.rmtree(
+                    destination
+                ) if destination.is_dir() else destination.unlink()
+        for destination, backup in backups:
+            os.replace(backup, destination)
+        raise
+    else:
+        for _destination, backup in backups:
+            if backup.exists():
+                shutil.rmtree(backup) if backup.is_dir() else backup.unlink()
+
+
+def prepare_docs_index_input(
+    contract: DocsEvidenceContract,
+    *,
+    repo_root: str | Path,
+    strict_tables: bool = False,
+    pandoc: str | Path | None = None,
+    allow_pipe_fallback: bool = False,
+) -> tuple[DocsIndexProfile, int]:
+    """Write the index-input mirror and manifest; return the profile and the number of normalised tables."""
+    root = Path(repo_root).resolve()
+    profile = resolve_docs_index_profile(contract)
+    retained = contract.retained_root.resolve()
+    before = _snapshot_retained(retained)
+
+    link_issues = validate_retained_source_links(contract, repo_root=root)
+    if link_issues:
+        raise ValueError("; ".join(link_issues))
+
+    manifest_text = _render_manifest(_mirror_exclude_globs(contract))
+    markdown_paths = _retained_markdown_paths(contract)
+
+    contract.staged_root.parent.mkdir(parents=True, exist_ok=True)
+    contract.staged_root.mkdir(parents=True, exist_ok=True)
+    temporary = Path(
+        tempfile.mkdtemp(prefix=".index-input-", dir=contract.staged_root.parent)
+    )
+    temp_input = temporary / INDEX_INPUT_NAME
+    temp_manifest = temporary / MANIFEST_NAME
+    tables_normalized = 0
+
+    try:
+        temp_input.mkdir(parents=True, exist_ok=True)
+        for source_path in markdown_paths:
+            relative = source_path.relative_to(retained)
+            original = source_path.read_text(encoding="utf-8")
+            transformed = augment_document_with_embedding_text(
+                original,
+                document_path=relative.as_posix(),
+                strict=strict_tables,
+                pandoc=pandoc,
+                allow_pipe_fallback=allow_pipe_fallback,
+            )
+            tables_normalized += len(transformed.tables)
+            destination = temp_input / relative
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            destination.write_text(transformed.markdown, encoding="utf-8")
+            _copy_resource_files(
+                source_markdown=source_path,
+                retained_root=retained,
+                mirror_root=temp_input,
+                original_markdown=original,
+            )
+
+        temp_manifest.write_text(manifest_text, encoding="utf-8")
+
+        _atomic_promote_pair(
+            (
+                (temp_input, profile.input_root),
+                (temp_manifest, profile.manifest_path),
+            )
+        )
+    finally:
+        if temporary.exists():
+            shutil.rmtree(temporary)
+        _assert_retained_unchanged(retained, before)
+
+    return profile, tables_normalized
+
+
+def sync_docs_index(
+    contract: DocsEvidenceContract,
+    *,
+    repo_root: str | Path,
+    options: SyncOptions | None = None,
+    embedder=None,
+    strict_tables: bool = False,
+    pandoc: str | Path | None = None,
+    allow_pipe_fallback: bool = False,
+) -> DocsIndexResult:
+    profile, tables_normalized = prepare_docs_index_input(
+        contract,
+        repo_root=repo_root,
+        strict_tables=strict_tables,
+        pandoc=pandoc,
+        allow_pipe_fallback=allow_pipe_fallback,
+    )
+    from ..evidence import sync as sync_module
+
+    report = sync_module.sync(
+        profile.input_root,
+        profile.manifest_path,
+        profile.database_path,
+        options=options,
+        embedder=embedder,
+        model_cache_dir=profile.model_cache_dir,
+    )
+    return DocsIndexResult(profile, tables_normalized, report)
+
+
+def search_docs_index(
+    contract: DocsEvidenceContract,
+    query: str,
+    *,
+    limit: int = 10,
+    fts_limit: int = 50,
+    vector_limit: int = 50,
+    embedder=None,
+) -> list[dict]:
+    from ..evidence import schema
+    from ..evidence.commands import active_embedding_model, hybrid_search
+
+    profile = resolve_docs_index_profile(contract)
+    if not profile.database_path.is_file():
+        return []
+    conn = schema.connect_existing(profile.database_path, read_only=True)
+    try:
+        schema.ensure_schema(conn)
+        embedder = embedder or active_embedding_model(conn, profile.model_cache_dir)
+        return hybrid_search(conn, query, limit, fts_limit, vector_limit, embedder)
+    finally:
+        conn.close()
+
+
+def _index_input_freshness(
+    contract: DocsEvidenceContract, profile: DocsIndexProfile
+) -> tuple[str, ...]:
+    diagnostics: list[str] = []
+    if not profile.input_root.is_dir():
+        diagnostics.append("index-input mirror is missing")
+        return tuple(diagnostics)
+    if not profile.manifest_path.is_file():
+        diagnostics.append("index-input manifest is missing")
+    expected_paths = {
+        path.relative_to(contract.retained_root).as_posix()
+        for path in _retained_markdown_paths(contract)
+    }
+    actual_paths = {
+        path.relative_to(profile.input_root).as_posix()
+        for path in sorted(profile.input_root.rglob("*.md"))
+        if path.is_file()
+    }
+    for path in sorted(expected_paths - actual_paths):
+        diagnostics.append(f"index-input missing: {path}")
+    for path in sorted(actual_paths - expected_paths):
+        diagnostics.append(f"index-input extra: {path}")
+    for relative in sorted(expected_paths & actual_paths):
+        source = contract.retained_root / relative
+        mirror = profile.input_root / relative
+        expected = augment_document_with_embedding_text(
+            source.read_text(encoding="utf-8"),
+            document_path=relative,
+        ).markdown
+        if mirror.read_text(encoding="utf-8") != expected:
+            diagnostics.append(f"index-input stale (retained changed): {relative}")
+    for path in sorted(profile.input_root.rglob("*")):
+        if not path.is_file():
+            continue
+        relative = path.relative_to(profile.input_root)
+        if any(part in _EXCLUDED_DIR_NAMES for part in relative.parts):
+            diagnostics.append(f"index-input leaked artifact: {relative.as_posix()}")
+        if path.suffix.lower() in _EXCLUDED_SUFFIXES:
+            diagnostics.append(f"index-input leaked artifact: {relative.as_posix()}")
+    return tuple(diagnostics)
+
+
+def _manifest_freshness(
+    contract: DocsEvidenceContract, profile: DocsIndexProfile
+) -> tuple[str, ...]:
+    if not profile.manifest_path.is_file():
+        return ()
+    expected = _render_manifest(_mirror_exclude_globs(contract))
+    try:
+        actual = profile.manifest_path.read_text(encoding="utf-8")
+    except OSError as exc:
+        return (f"index-input manifest is unreadable: {exc}",)
+    if actual != expected:
+        return (
+            "index-input manifest is stale or does not match the current docs contract",
+        )
+    return ()
+
+
+def _database_freshness(profile: DocsIndexProfile) -> tuple[str, ...]:
+    if not profile.manifest_path.is_file():
+        return ()
+    from ..evidence import manifest, schema
+
+    try:
+        rules = manifest.load_manifest(profile.manifest_path)
+    except Exception:  # noqa: BLE001
+        # An unparsable manifest is already reported by _manifest_freshness;
+        # there is no usable source selection to compare the database against.
+        return ()
+    discovered = manifest.resolve_sources(profile.input_root, rules).selected
+    conn = schema.connect_existing(profile.database_path, read_only=True)
+    try:
+        schema.ensure_schema(conn)
+        rows = {
+            row["path"]: row["sha256"]
+            for row in conn.execute("SELECT path, sha256 FROM sources")
+        }
+    finally:
+        conn.close()
+    diagnostics: list[str] = []
+    for path in sorted(set(discovered) - set(rows)):
+        diagnostics.append(f"documentation index database missing source: {path}")
+    for path in sorted(set(rows) - set(discovered)):
+        diagnostics.append(f"documentation index database has stale source: {path}")
+    for path in sorted(set(discovered) & set(rows)):
+        if file_sha256(discovered[path]) != rows[path]:
+            diagnostics.append(
+                f"documentation index database is stale relative to mirror: {path}"
+            )
+    return tuple(diagnostics)
+
+
+def docs_index_freshness_diagnostics(contract: DocsEvidenceContract) -> tuple[str, ...]:
+    profile = resolve_docs_index_profile(contract)
+    diagnostics = list(_index_input_freshness(contract, profile))
+    diagnostics.extend(_manifest_freshness(contract, profile))
+    if not profile.database_path.is_file():
+        diagnostics.append("documentation index database is missing")
+    else:
+        diagnostics.extend(_database_freshness(profile))
+    return tuple(diagnostics)
+
+
+def doctor_docs_index(contract: DocsEvidenceContract) -> tuple[str, ...]:
+    profile = resolve_docs_index_profile(contract)
+    issues = list(docs_index_freshness_diagnostics(contract))
+    if not profile.database_path.is_file():
+        return tuple(issues)
+    from ..evidence import schema
+    from ..evidence import sync as sync_module
+
+    conn = schema.connect_existing(profile.database_path, read_only=True)
+    try:
+        schema.ensure_schema(conn)
+        issues.extend(sync_module.integrity_check(conn))
+    finally:
+        conn.close()
+    return tuple(issues)

tracecite/evidence/__init__.py

@@ -0,0 +1,3 @@
+"""Optional TraceCite evidence indexing and verification subsystem."""
+
+from __future__ import annotations

tracecite/evidence/chunking.py

@@ -0,0 +1,214 @@
+"""Greedy char-budget chunker, normalisation, and chunk-identity hashing.
+
+Parsers emit fine-grained ``ParsedChunkUnit`` sequences (for example, a PDF
+text block, Markdown paragraph, or workbook row range). This module groups
+adjacent units into retrieval-sized chunks and computes two identity
+hashes:
+
+- ``lexical_hash``: sha256 of the raw (unnormalised) chunk body.
+- ``semantic_input_hash``: sha256 of
+  ``normalisation_version + source_type + semantic_context + normalised body``.
+  Path, line, page, and modification time are locators, not meaning, and are
+  deliberately excluded.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import dataclass, field
+
+from .parsers.base import ParsedChunkUnit
+
+DEFAULT_MAX_CHUNK_CHARS = 1200
+_CELL_REFERENCE_RE = re.compile(r"^([A-Z]+)([1-9][0-9]*)$")
+
+
+@dataclass(frozen=True)
+class ChunkCandidate:
+    logical_key: str
+    ordinal: int
+    heading_path: list[str]
+    symbol: str | None
+    body: str
+    content_type: str
+    physical_page: int | None
+    page_start_offset: int | None
+    page_end_offset: int | None
+    page_range_start: int | None
+    page_range_end: int | None
+    line_start: int | None
+    line_end: int | None
+    locator: dict = field(default_factory=dict)
+    semantic_input_hash: str = ""
+    lexical_hash: str = ""
+
+
+def normalise_text(text: str) -> str:
+    """Collapse whitespace so trivial reflow never changes semantic identity."""
+
+    return " ".join(text.split())
+
+
+def compute_lexical_hash(body: str) -> str:
+    return hashlib.sha256(body.encode("utf-8")).hexdigest()
+
+
+def compute_semantic_input_hash(
+    *,
+    normalisation_version: str,
+    source_type: str,
+    heading_path: list[str],
+    symbol: str | None,
+    body: str,
+) -> str:
+    normalised_body = normalise_text(body)
+    context = json.dumps([heading_path, symbol], sort_keys=True)
+    payload = (
+        f"{normalisation_version}\x1f{source_type}\x1f{context}\x1f{normalised_body}"
+    )
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def group_units(
+    units: list[ParsedChunkUnit], max_chars: int = DEFAULT_MAX_CHUNK_CHARS
+) -> list[list[ParsedChunkUnit]]:
+    """Greedily group consecutive units into chunks bounded by ``max_chars``.
+
+    A single unit is never split. Grouping only combines adjacent units that
+    share the same heading path so a chunk never silently straddles a
+    section boundary.
+    """
+
+    groups: list[list[ParsedChunkUnit]] = []
+    current: list[ParsedChunkUnit] = []
+    current_len = 0
+    for unit in units:
+        unit_len = len(unit.text)
+        breaks_group = current and (
+            current_len + unit_len > max_chars
+            or unit.heading_path != current[-1].heading_path
+            or unit.physical_page != current[-1].physical_page
+        )
+        if breaks_group:
+            groups.append(current)
+            current = []
+            current_len = 0
+        current.append(unit)
+        current_len += unit_len + 2
+    if current:
+        groups.append(current)
+    return groups
+
+
+def _cell_coordinates(reference: str) -> tuple[int, int] | None:
+    match = _CELL_REFERENCE_RE.fullmatch(reference)
+    if match is None:
+        return None
+    column = 0
+    for character in match.group(1):
+        column = column * 26 + ord(character) - ord("A") + 1
+    return column, int(match.group(2))
+
+
+def _column_label(column: int) -> str:
+    label = ""
+    while column:
+        column, remainder = divmod(column - 1, 26)
+        label = chr(ord("A") + remainder) + label
+    return label
+
+
+def _excel_bounding_range(locators: list[dict]) -> str | None:
+    coordinates: list[tuple[int, int]] = []
+    for locator in locators:
+        for key in ("start_cell", "end_cell"):
+            parsed = _cell_coordinates(str(locator.get(key, "")))
+            if parsed is None:
+                return None
+            coordinates.append(parsed)
+    min_column = min(column for column, _row in coordinates)
+    max_column = max(column for column, _row in coordinates)
+    min_row = min(row for _column, row in coordinates)
+    max_row = max(row for _column, row in coordinates)
+    start = f"{_column_label(min_column)}{min_row}"
+    end = f"{_column_label(max_column)}{max_row}"
+    return start if start == end else f"{start}:{end}"
+
+
+def _merged_locator(group: list[ParsedChunkUnit]) -> dict:
+    locators = [unit.locator for unit in group if unit.locator]
+    if not locators:
+        return {"unit_count": len(group)}
+    if all(locator.get("kind") == "excel-range" for locator in locators):
+        sheets = {locator.get("sheet") for locator in locators}
+        if len(sheets) == 1:
+            first = locators[0]
+            exact_ranges = [locator["range"] for locator in locators]
+            return {
+                "kind": "excel-range",
+                "sheet": first.get("sheet"),
+                "sheet_index": first.get("sheet_index"),
+                "sheet_state": first.get("sheet_state"),
+                "range": _excel_bounding_range(locators),
+                "range_kind": "bounding",
+                "exact_ranges": exact_ranges,
+                "row_start": min(locator["row_start"] for locator in locators),
+                "row_end": max(locator["row_end"] for locator in locators),
+                "unit_count": len(group),
+            }
+    if len(locators) == 1:
+        return {**locators[0], "unit_count": len(group)}
+    return {"unit_count": len(group), "parts": locators}
+
+
+def build_chunk_candidates(
+    groups: list[list[ParsedChunkUnit]],
+    *,
+    source_type: str,
+    normalisation_version: str,
+) -> list[ChunkCandidate]:
+    candidates: list[ChunkCandidate] = []
+    for ordinal, group in enumerate(groups):
+        first = group[0]
+        last = group[-1]
+        body = "\n\n".join(unit.text for unit in group)
+        content_types = {unit.content_type for unit in group}
+        content_type = first.content_type if len(content_types) == 1 else "mixed"
+
+        physical_pages = {
+            unit.physical_page for unit in group if unit.physical_page is not None
+        }
+        physical_page = first.physical_page if len(physical_pages) <= 1 else None
+        page_range_start = min(physical_pages) if physical_pages else None
+        page_range_end = max(physical_pages) if physical_pages else None
+
+        semantic_hash = compute_semantic_input_hash(
+            normalisation_version=normalisation_version,
+            source_type=source_type,
+            heading_path=first.heading_path,
+            symbol=first.symbol,
+            body=body,
+        )
+        candidates.append(
+            ChunkCandidate(
+                logical_key=first.logical_key,
+                ordinal=ordinal,
+                heading_path=first.heading_path,
+                symbol=first.symbol,
+                body=body,
+                content_type=content_type,
+                physical_page=physical_page,
+                page_start_offset=first.page_start_offset,
+                page_end_offset=last.page_end_offset,
+                page_range_start=page_range_start,
+                page_range_end=page_range_end,
+                line_start=first.line_start,
+                line_end=last.line_end,
+                locator=_merged_locator(group),
+                semantic_input_hash=semantic_hash,
+                lexical_hash=compute_lexical_hash(body),
+            )
+        )
+    return candidates