okfgraph 0.2.4__py3-none-any.whl

okfgraph/components/__init__.py

@@ -0,0 +1,61 @@
+"""OKFRouter component objects.
+
+These are extracted from the monolithic ``okfgraph.router.OKFRouter`` as part
+of the Phase 0–4 refactor (see ``docs/plan-router-refactor-components.md``).
+Each component receives its dependencies explicitly via ``__init__`` (dependency
+injection) rather than reaching into a shared ``self``.
+
+Phase 0 status: classes and method signatures are scaffolded; method bodies are
+stubs (``...``) until their respective phase moves the implementation over.
+``lint`` is fully implemented (self-contained, no router state required).
+"""
+
+from okfgraph.components.schema import SchemaManager
+from okfgraph.components.delta import DeltaDetector
+from okfgraph.components.purge import PurgeManager
+from okfgraph.components.embedding import EmbeddingEngine
+from okfgraph.components.image_assets import ImageAssetManager
+from okfgraph.components.search import SearchEngine
+from okfgraph.components.import_ import ImportManager, parse_source_file
+from okfgraph.components.export import ExportManager
+from okfgraph.components.ingest import IngestManager
+from okfgraph.components.converters import BobineConverter, ConvertedDocument
+from okfgraph.components.ranking import ppr, seed_ranked_ppr, seeds
+from okfgraph.components.links import (
+    build_name_index,
+    extract_md_links,
+    extract_wikilinks,
+    normalize_path_link,
+    resolve_wiki,
+)
+from okfgraph.components.diff import DiffManager, DiffState, state_of_dir
+from okfgraph.components.doctor import DoctorManager
+from okfgraph.components.lint import lint_bundle
+
+__all__ = [
+    "SchemaManager",
+    "DeltaDetector",
+    "PurgeManager",
+    "EmbeddingEngine",
+    "ImageAssetManager",
+    "SearchEngine",
+    "ImportManager",
+    "ExportManager",
+    "IngestManager",
+    "BobineConverter",
+    "ConvertedDocument",
+    "parse_source_file",
+    "seeds",
+    "ppr",
+    "seed_ranked_ppr",
+    "build_name_index",
+    "extract_md_links",
+    "extract_wikilinks",
+    "normalize_path_link",
+    "resolve_wiki",
+    "DiffManager",
+    "DiffState",
+    "state_of_dir",
+    "DoctorManager",
+    "lint_bundle",
+]

okfgraph/components/converters.py

@@ -0,0 +1,128 @@
+"""Document-converter interface for PDF ingestion.
+
+The knowledge graph never talks to a conversion engine directly — it talks
+to a :class:`DocumentConverter`. Bobine is just the default implementation;
+any pipeline that turns a PDF into markdown plus staged images can plug in
+by implementing the one-method protocol::
+
+    class MyConverter:
+        def convert(self, pdf_path, output_dir, *, on_page=None):
+            ...
+            return ConvertedDocument(md_path=..., image_dir=..., page_count=...)
+
+Device selection for ONNX-backed converters is ORT-level
+(``ORT_DYLIB_PATH``) and intentionally not part of this interface.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Optional, Protocol
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ConvertedDocument:
+    """The only thing the graph needs from a conversion pipeline.
+
+    Attributes:
+        md_path: The converted ``<stem>.md`` file inside ``output_dir``.
+            Transient when produced in a temp dir (auto-import) — the graph
+            keeps the content, not the file.
+        image_dir: Directory holding the staged images (may be empty).
+        page_count: Number of pages in the source PDF.
+    """
+
+    md_path: Path
+    image_dir: Path
+    page_count: int
+
+
+class DocumentConverter(Protocol):
+    """One-method protocol every ingestion pipeline must satisfy."""
+
+    def convert(
+        self,
+        pdf_path: str | Path,
+        output_dir: str | Path,
+        *,
+        on_page: Callable[[int, int], None] | None = None,
+    ) -> ConvertedDocument:
+        """Convert ``pdf_path`` into ``output_dir``.
+
+        Must write ``<stem>.md`` plus any staged images into ``output_dir``
+        and return their locations. ``on_page`` receives 0-based
+        ``(page_index, page_total)`` progress callbacks.
+        """
+        ...  # pragma: no cover - protocol stub
+
+
+# Bobine routing names are Capitalized; the public API takes lowercase.
+_BOBINE_ROUTING = {
+    "auto": "Auto",
+    "surgical": "Surgical",
+    "always": "Always",
+    "never": "Never",
+}
+
+
+class BobineConverter:
+    """The bobine (Rust) implementation of :class:`DocumentConverter`.
+
+    Args:
+        routing_mode: "auto" | "surgical" | "always" | "never". Controls
+            when bobine invokes ONNX models ("never" = pdf_oxide fast path).
+        extract_images: Whether to extract embedded images from the PDF.
+
+    Raises:
+        ValueError: On an unknown routing_mode (at construction — no work done).
+        RuntimeError: When converting without bobine installed.
+    """
+
+    def __init__(self, routing_mode: str = "auto", extract_images: bool = True):
+        if routing_mode not in _BOBINE_ROUTING:
+            raise ValueError(
+                f"routing_mode must be one of {sorted(_BOBINE_ROUTING)}, "
+                f"got '{routing_mode}'"
+            )
+        self.routing_mode = routing_mode
+        self.extract_images = extract_images
+
+    def convert(
+        self,
+        pdf_path: str | Path,
+        output_dir: str | Path,
+        *,
+        on_page: Callable[[int, int], None] | None = None,
+    ) -> ConvertedDocument:
+        try:
+            import bobine  # noqa: PLC0415 — optional dependency, lazy import
+        except ImportError:
+            raise RuntimeError(
+                "bobine is required for PDF ingestion: pip install bobine"
+            ) from None
+
+        mode = getattr(bobine.RoutingMode, _BOBINE_ROUTING[self.routing_mode])
+        config = bobine.ConverterConfig(
+            routing_mode=mode,
+            extract_images=self.extract_images,
+        )
+        doc = bobine.ingest_document(
+            str(pdf_path),
+            str(output_dir),
+            config,
+            should_continue=lambda: True,
+            on_page=on_page
+            or (lambda idx, total: logger.info("page %d/%d", idx + 1, total)),
+        )
+        return ConvertedDocument(
+            md_path=Path(doc.md_path),
+            image_dir=Path(doc.image_dir),
+            page_count=doc.page_count,
+        )
+
+
+__all__ = ["ConvertedDocument", "DocumentConverter", "BobineConverter"]

okfgraph/components/delta.py

@@ -0,0 +1,271 @@
+"""Change-detection (directory/file hashing) extracted during the OKFRouter Phase 1 refactor.
+
+Bodies are verbatim from okfgraph/router.py; the facade (OKFRouter) owns
+the shared resources (conn, embedder, tokenizer, ...) and injects them
+here. Public callers reach these via router.<method> (component bridge).
+"""
+import hashlib
+import json
+import logging
+from pathlib import Path
+from typing import Dict, List
+
+from okfgraph.components.import_ import is_concept_file
+
+logger = logging.getLogger(__name__)
+
+class DeltaDetector:
+    """Detects which source files/directories changed since last ingest."""
+
+    SUPPORTED_SOURCE_EXTS = (".md", ".markdown", ".txt")
+
+    def __init__(self, conn, bundle_root):
+        self.conn = conn
+        self.bundle_root = bundle_root
+
+    def _file_hash(self, file_path: Path) -> str:
+        """SHA-256 hex digest of a file's raw bytes."""
+        return hashlib.sha256(file_path.read_bytes()).hexdigest()
+
+
+    def _compute_directory_hash(self, dir_path: Path) -> str:
+        """Compute a combined hash for a directory's contents.
+
+        Hashes all .md/.txt files in the directory (recursively) sorted by relative path,
+        then hashes the concatenation of all file hashes. Returns a single
+        SHA-256 hex digest that changes if any file in the subtree changes.
+        """
+        file_hashes = []
+        for fp in sorted(dir_path.rglob("*")):
+            if is_concept_file(fp):
+                rel = str(fp.relative_to(dir_path))
+                fh = self._file_hash(fp)
+                file_hashes.append((rel, fh))
+        combined = "|".join(f"{rel}:{fh}" for rel, fh in file_hashes)
+        return hashlib.sha256(combined.encode("utf-8")).hexdigest()
+
+
+    def _compute_directory_hash_with_files(self, dir_path: Path) -> tuple[str, List[str]]:
+        """Compute a combined hash for a directory's contents and return file paths.
+
+        Returns (hash, [relative_file_paths]) where file_paths can be used to
+        identify deleted files when the directory is removed.
+        """
+        file_hashes = []
+        file_paths = []
+        for fp in sorted(dir_path.rglob("*")):
+            if is_concept_file(fp):
+                rel = str(fp.relative_to(dir_path))
+                fh = self._file_hash(fp)
+                file_hashes.append((rel, fh))
+                file_paths.append(rel)
+        combined = "|".join(f"{rel}:{fh}" for rel, fh in file_hashes)
+        return hashlib.sha256(combined.encode("utf-8")).hexdigest(), file_paths
+
+
+    def _load_directory_hashes(self) -> Dict[str, Dict]:
+        """Load the persisted directory→{hash, files} mapping, or return empty dict."""
+        try:
+            rows = self.conn.execute(
+                "MATCH (d:DirHash) RETURN d.path AS p, d.hash AS h, d.files AS f"
+            ).rows_as_dict().get_all()
+            if rows:
+                result = {}
+                for r in rows:
+                    files_str = r.get("f") or ""
+                    try:
+                        files = json.loads(files_str) if files_str else []
+                    except (json.JSONDecodeError, TypeError):
+                        files = []
+                    result[r["p"]] = {"hash": r["h"], "files": files}
+                return result
+        except Exception as exc:
+            logger.debug("could not load directory hashes: %s", exc)
+        return {}
+
+
+    def _store_directory_hashes(self, hashes: Dict[str, Dict]) -> None:
+        """Persist a path→{hash, files} mapping in the DirHash table."""
+        try:
+            for path, data in hashes.items():
+                files_str = json.dumps(data.get("files", []))
+                self.conn.execute(
+                    """
+                    MERGE (d:DirHash {path: $p})
+                    SET d.hash = $h, d.files = $f
+                    """,
+                    {"p": path, "h": data["hash"], "f": files_str},
+                )
+        except Exception as exc:
+            logger.debug("could not store directory hashes: %s", exc)
+
+
+    def _changed_directories(self, source_files: List[Path]) -> tuple[List[Path], List[str]]:
+        """Return (changed_files, deleted_paths) using directory-level hash aggregation.
+
+        Groups files by parent directory, computes combined directory hashes,
+        and skips entire subtrees when the directory hash hasn't changed.
+        Only files in changed directories are returned as changed.
+        Deleted paths are relative file paths from the stored map of deleted directories.
+        """
+        stored_dir_hashes = self._load_directory_hashes()
+
+        # Group files by parent directory
+        dir_files: Dict[str, List[Path]] = {}
+        for fp in source_files:
+            parent = str(fp.parent.relative_to(self.bundle_root))
+            dir_files.setdefault(parent, []).append(fp)
+
+        # Compute current directory hashes with file paths
+        current_dir_hashes: Dict[str, Dict] = {}
+        changed: List[Path] = []
+        for dir_rel, files in dir_files.items():
+            dir_path = self.bundle_root / dir_rel
+            if dir_path.exists():
+                dir_hash, file_paths = self._compute_directory_hash_with_files(dir_path)
+            else:
+                dir_hash, file_paths = "", []
+            current_dir_hashes[dir_rel] = {"hash": dir_hash, "files": file_paths}
+
+            # Check if directory hash changed
+            stored_data = stored_dir_hashes.get(dir_rel, {})
+            stored_hash = stored_data.get("hash") if stored_data else None
+            if dir_hash != stored_hash:
+                # Directory changed — add all files in it
+                changed.extend(files)
+
+        # Detect deleted directories: dirs in stored map but not on disk
+        stored_dirs = set(stored_dir_hashes.keys())
+        current_dirs = set(current_dir_hashes.keys())
+        deleted_dirs = sorted(stored_dirs - current_dirs)
+
+        # Collect deleted file paths from deleted directories
+        deleted_paths: List[str] = []
+        for del_dir in deleted_dirs:
+            stored_data = stored_dir_hashes.get(del_dir, {})
+            stored_files = stored_data.get("files", []) if stored_data else []
+            for rel_file in stored_files:
+                # Use native path separator to match FileHash entries
+                deleted_paths.append(str(Path(del_dir) / rel_file))
+
+        # Persist the new directory hashes for the next run
+        self._store_directory_hashes(current_dir_hashes)
+
+        if changed:
+            logger.info(
+                "directory-delta: %d changed dir(s), %d changed files out of %d total",
+                len([d for d in dir_files if current_dir_hashes.get(d, {}).get("hash") != stored_dir_hashes.get(d, {}).get("hash")]),
+                len(changed),
+                len(source_files),
+            )
+        else:
+            logger.info("directory-delta: no changes detected")
+
+        if deleted_paths:
+            logger.info(
+                "directory-delta: %d deleted file(s) in %d deleted directory(s): %s",
+                len(deleted_paths),
+                len(deleted_dirs),
+                deleted_paths[:5],  # Limit output
+            )
+
+        return changed, deleted_paths
+
+
+    def _store_file_hashes(self, hashes: Dict[str, str]) -> None:
+        """Persist a path→hash mapping in the FileHash table.
+
+        Upserts each row so the table always reflects the current state.
+        Also stores the concept_id for each file to enable safe purge.
+        """
+        try:
+            for path, h in hashes.items():
+                # Derive concept_id from path: strip extension, normalise separators.
+                concept_id = str(Path(path).with_suffix("")).replace("\\", "/")  # remove .md / .txt suffix, use forward slashes
+                self.conn.execute(
+                    """
+                    MERGE (f:FileHash {path: $p})
+                    SET f.hash = $h, f.concept_id = $c
+                    """,
+                    {"p": path, "h": h, "c": concept_id},
+                )
+        except Exception as exc:
+            logger.debug("could not store file hashes: %s", exc)
+
+
+    def _load_file_hashes(self) -> Dict[str, str]:
+        """Load the persisted path→hash mapping, or return empty dict."""
+        try:
+            rows = self.conn.execute(
+                "MATCH (f:FileHash) RETURN f.path AS p, f.hash AS h"
+            ).rows_as_dict().get_all()
+            if rows:
+                return {r["p"]: r["h"] for r in rows}
+        except Exception as exc:
+            logger.debug("could not load file hashes: %s", exc)
+        return {}
+
+
+    def _load_file_hash_concept_ids(self) -> Dict[str, str]:
+        """Load the persisted path→concept_id mapping, or return empty dict."""
+        try:
+            rows = self.conn.execute(
+                "MATCH (f:FileHash) RETURN f.path AS p, f.concept_id AS c"
+            ).rows_as_dict().get_all()
+            if rows:
+                return {r["p"]: r["c"] for r in rows}
+        except Exception as exc:
+            logger.debug("could not load file hash concept ids: %s", exc)
+        return {}
+
+
+    def _changed_files(
+        self, source_files: List[Path]
+    ) -> tuple[List[Path], List[str]]:
+        """Return (changed_files, deleted_paths) since last import.
+
+        Compares SHA-256 of each file against the hashes persisted from the
+        previous ``import_bundle()`` call.  New files (not in the stored map)
+        are treated as changed.  Files in the stored map but absent from disk
+        are returned as deleted_paths.
+
+        The stored map is updated after the check.
+        """
+        stored = self._load_file_hashes()
+
+        current: Dict[str, str] = {}
+        changed: List[Path] = []
+        for fp in source_files:
+            rel = str(fp.relative_to(self.bundle_root))
+            h = self._file_hash(fp)
+            current[rel] = h
+            if h != stored.get(rel):
+                changed.append(fp)
+
+        # Detect deleted files: paths in stored map but not on disk.
+        stored_paths = set(stored.keys())
+        current_paths = set(current.keys())
+        deleted = sorted(stored_paths - current_paths)
+
+        # Persist the new mapping for the next run.
+        self._store_file_hashes(current)
+
+        if changed:
+            logger.info(
+                "delta: %d changed / %d total files",
+                len(changed),
+                len(source_files),
+            )
+        else:
+            logger.info("delta: no changes detected")
+
+        if deleted:
+            logger.info(
+                "delta: %d deleted files detected: %s",
+                len(deleted),
+                deleted,
+            )
+
+        return changed, deleted
+
+

okfgraph/components/diff.py

@@ -0,0 +1,172 @@
+"""Structural diff: knowledge-structure comparison, not text hunks.
+
+Compares two states — a bundle directory (parsed, never imported) or the live
+graph — and reports concepts added/removed/changed, retitles/retypes, edge
+deltas, and newly broken vs fixed links. Pure hash/set comparison; every list
+sorted. Exit-code friendly (``identical`` flag for CI gates).
+
+Drift mode (graph vs its bundle dir) answers "what changed since last import";
+snapshot mode (dir vs dir) previews an import or diffs two vaults.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Set, Tuple
+
+from okfgraph.components.import_ import is_concept_file, parse_source_file
+from okfgraph.components.links import (
+    build_name_index,
+    extract_md_links,
+    extract_wikilinks,
+    is_external,
+    normalize_path_link,
+    resolve_wiki,
+)
+
+logger = logging.getLogger(__name__)
+
+# NOTE: file enumeration uses is_concept_file() from import_ (extensions +
+# reserved-name skip) so diff and import agree on what a concept file is.
+
+
+def content_hash(body: str) -> str:
+    """SHA-256 over newline-normalized, stripped body text."""
+    norm = (body or "").replace("\r\n", "\n").strip()
+    return hashlib.sha256(norm.encode("utf-8")).hexdigest()
+
+
+@dataclass
+class DiffState:
+    """One diffable side: concept meta, resolved edges, unresolved links."""
+
+    concepts: Dict[str, Dict[str, Any]] = field(default_factory=dict)
+    edges: Set[Tuple[str, str]] = field(default_factory=set)
+    broken: Set[Tuple[str, str]] = field(default_factory=set)
+
+
+def state_of_dir(bundle_dir: Path) -> DiffState:
+    """Parse a bundle directory into a DiffState (no database, no import)."""
+    state = DiffState()
+    files = sorted(fp for fp in Path(bundle_dir).rglob("*") if is_concept_file(fp))
+    raw_links: Dict[str, Dict[str, List[str]]] = {}
+    index_concepts = []
+    for fp in files:
+        try:
+            concept, body, cid = parse_source_file(fp, Path(bundle_dir))
+        except Exception as e:
+            logger.warning("diff: skipping unparsable %s: %s", fp, e)
+            continue
+        extra = concept.model_extra or {}
+        state.concepts[cid] = {
+            "title": concept.title,
+            "type": concept.type,
+            "hash": content_hash(body),
+        }
+        index_concepts.append({
+            "id": cid,
+            "title": concept.title,
+            "uid": extra.get("uid"),
+            "aliases": extra.get("aliases"),
+            "alias": extra.get("alias"),
+        })
+        raw_links[cid] = {
+            "md": extract_md_links(body),
+            "wiki": extract_wikilinks(body),
+        }
+    maps, _ = build_name_index(index_concepts)
+    known = set(state.concepts)
+    for cid, links in raw_links.items():
+        for raw in links["md"]:
+            if is_external(raw):
+                continue
+            target = normalize_path_link(raw.split("#", 1)[0])
+            if target in known:
+                state.edges.add((cid, target))
+            else:
+                state.broken.add((cid, target))
+        for raw in links["wiki"]:
+            if not raw or is_external(raw):
+                continue
+            target = resolve_wiki(raw, maps, known)
+            if target is not None:
+                state.edges.add((cid, target))
+            else:
+                state.broken.add((cid, raw))
+    return state
+
+
+class DiffManager:
+    """Diffs involving the live graph (dir-vs-dir uses ``state_of_dir``)."""
+
+    def __init__(self, conn):
+        self.conn = conn
+
+    def state_of_db(self) -> DiffState:
+        """Read the live graph into a DiffState."""
+        state = DiffState()
+        for r in self.conn.execute(
+            "MATCH (c:Concept) RETURN c.id, c.title, c.type, c.body"
+        ).rows_as_dict().get_all():
+            state.concepts[r["c.id"]] = {
+                "title": r.get("c.title"),
+                "type": r.get("c.type"),
+                "hash": content_hash(r.get("c.body") or ""),
+            }
+        for r in self.conn.execute(
+            "MATCH (a:Concept)-[:LINKS_TO]->(b:Concept) "
+            "RETURN a.id AS src, b.id AS dst"
+        ).rows_as_dict().get_all():
+            state.edges.add((r["src"], r["dst"]))
+        for r in self.conn.execute(
+            "MATCH (bl:BrokenLink) RETURN bl.source_id AS source, bl.target_id AS target"
+        ).rows_as_dict().get_all():
+            state.broken.add((r["source"], r["target"]))
+        return state
+
+    @staticmethod
+    def compare(a: DiffState, b: DiffState) -> Dict[str, Any]:
+        """Pure comparison of two states (a = old, b = new). All lists sorted."""
+        a_ids, b_ids = set(a.concepts), set(b.concepts)
+        added = sorted(b_ids - a_ids)
+        removed = sorted(a_ids - b_ids)
+        changed, retitled, retyped = [], [], []
+        for cid in sorted(a_ids & b_ids):
+            old, new = a.concepts[cid], b.concepts[cid]
+            if old["hash"] != new["hash"]:
+                changed.append(cid)
+            if (old["title"] or "") != (new["title"] or ""):
+                retitled.append({"id": cid, "old": old["title"], "new": new["title"]})
+            if (old["type"] or "") != (new["type"] or ""):
+                retyped.append({"id": cid, "old": old["type"], "new": new["type"]})
+        edges_added = sorted(b.edges - a.edges)
+        edges_removed = sorted(a.edges - b.edges)
+        broken_new = sorted(b.broken - a.broken)
+        broken_fixed = sorted(a.broken - b.broken)
+        identical = not (
+            added or removed or changed or retitled or retyped
+            or edges_added or edges_removed or broken_new or broken_fixed
+        )
+        return {
+            "added": added,
+            "removed": removed,
+            "changed": changed,
+            "retitled": retitled,
+            "retyped": retyped,
+            "edges_added": [list(e) for e in edges_added],
+            "edges_removed": [list(e) for e in edges_removed],
+            "broken_new": [list(e) for e in broken_new],
+            "broken_fixed": [list(e) for e in broken_fixed],
+            "identical": identical,
+        }
+
+    def diff_dirs(self, old: Path, new: Path) -> Dict[str, Any]:
+        """Snapshot mode: two bundle directories, no database needed."""
+        return self.compare(state_of_dir(old), state_of_dir(new))
+
+    def diff_db_dir(self, bundle_dir: Path) -> Dict[str, Any]:
+        """Drift mode: live graph (old) vs bundle directory (new)."""
+        return self.compare(self.state_of_db(), state_of_dir(bundle_dir))