treesearchlib 1.1.0__py3-none-any.whl

treesearch/fts.py

@@ -0,0 +1,2293 @@
+# -*- coding: utf-8 -*-
+"""
+@author:XuMing(xuming624@qq.com)
+@description: SQLite FTS5 full-text search engine for tree-structured documents.
+
+Single-file storage: tree structures, FTS5 indexes, and incremental metadata
+are all stored in one SQLite database (.db file).
+
+Architecture: "SQLite FTS5 + Producer-Consumer"
+  - Deferred indexing via WAL mode solves real-time freshness
+  - Local SQL execution handles aggregation needs
+  - FTS5 inverted index guarantees retrieval performance
+  - Tree structure persistence in `documents.structure_json` column
+
+Key features:
+  - WAL mode for concurrent read/write
+  - Lazy indexing: nodes are inserted on demand, not precomputed
+  - MD-aware schema: front_matter, title, summary, body, code_blocks
+  - CJK-aware tokenizer: jieba segmentation only when Chinese text is detected
+  - Implements PreFilter protocol for seamless integration with search()
+  - Hierarchical field boosting via FTS5 column weighting
+"""
+import hashlib
+import json
+import logging
+import os
+import re
+import sqlite3
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# FTS5 column weights: title > body > summary > code
+# Used in bm25() ranking function
+_DEFAULT_WEIGHTS = {
+    "title": 5.0,
+    "summary": 2.0,
+    "body": 10.0,
+    "code_blocks": 1.0,
+    "front_matter": 2.0,
+}
+
+# ---------------------------------------------------------------------------
+# FTS5 availability detection
+# ---------------------------------------------------------------------------
+
+_FTS5_AVAILABLE: Optional[bool] = None
+
+
+class _NullContext:
+    """No-op transaction context for callers managing commits externally."""
+    def __enter__(self):
+        return self
+    def __exit__(self, *exc):
+        return False
+
+
+def _check_fts5() -> bool:
+    """Check whether the current SQLite build includes the FTS5 extension."""
+    global _FTS5_AVAILABLE
+    if _FTS5_AVAILABLE is not None:
+        return _FTS5_AVAILABLE
+    try:
+        conn = sqlite3.connect(":memory:")
+        conn.execute("CREATE VIRTUAL TABLE _fts5_test USING fts5(x)")
+        conn.execute("DROP TABLE _fts5_test")
+        conn.close()
+        _FTS5_AVAILABLE = True
+    except sqlite3.OperationalError:
+        _FTS5_AVAILABLE = False
+        logger.warning(
+            "SQLite FTS5 extension is not available (SQLite version: %s). "
+            "Falling back to plain-text LIKE search. Performance and ranking "
+            "quality will be reduced. To fix: upgrade SQLite or install a "
+            "Python build that includes FTS5.",
+            sqlite3.sqlite_version,
+        )
+    return _FTS5_AVAILABLE
+
+
+def _sqlite_regexp(pattern: str, string: str) -> bool:
+    """SQLite REGEXP callback — registered via create_function."""
+    if string is None or pattern is None:
+        return False
+    try:
+        return bool(re.search(pattern, string))
+    except re.error:
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Markdown structure parser
+# ---------------------------------------------------------------------------
+
+_RE_FRONT_MATTER = re.compile(r"^---\s*\n(.*?\n)---\s*\n", re.DOTALL)
+_RE_CODE_BLOCK = re.compile(r"```[\w]*\n(.*?)```", re.DOTALL)
+_RE_HEADING_LINE = re.compile(r"^#{1,6}\s+")
+
+
+def parse_md_node_text(text: str) -> dict:
+    """Parse a node's text into MD-aware structured fields.
+
+    Returns:
+        {
+            "front_matter": str,  # YAML front matter (if present)
+            "body": str,          # main text (headings, paragraphs)
+            "code_blocks": str,   # concatenated code blocks
+        }
+    """
+    if not text:
+        return {"front_matter": "", "body": "", "code_blocks": ""}
+
+    front_matter = ""
+    remaining = text
+
+    # Extract front matter
+    fm_match = _RE_FRONT_MATTER.match(text)
+    if fm_match:
+        front_matter = fm_match.group(1).strip()
+        remaining = text[fm_match.end():]
+
+    # Extract code blocks
+    code_parts = []
+    def _replace_code(m):
+        code_parts.append(m.group(1).strip())
+        return ""  # remove from body
+    body = _RE_CODE_BLOCK.sub(_replace_code, remaining)
+    code_blocks = "\n".join(code_parts)
+
+    # Clean body: collapse blank lines
+    body = re.sub(r"\n{3,}", "\n\n", body).strip()
+
+    return {
+        "front_matter": front_matter,
+        "body": body,
+        "code_blocks": code_blocks,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Tokenizer for FTS5 (Chinese/English)
+# ---------------------------------------------------------------------------
+
+from functools import lru_cache
+
+from .tokenizer import _RE_HAS_CJK
+
+
+@lru_cache(maxsize=4096)
+def _tokenize_for_fts(text: str) -> str:
+    """Tokenize text for FTS5 indexing. Space-separated tokens.
+
+    Only uses jieba segmentation when Chinese (CJK) characters are detected.
+    For pure English/non-CJK text, relies on FTS5's built-in unicode61 tokenizer
+    (no jieba overhead).
+
+    Results are LRU-cached to avoid repeated jieba overhead for the same text.
+    """
+    if not text or not text.strip():
+        return ""
+    if _RE_HAS_CJK.search(text):
+        from .tokenizer import tokenize
+        tokens = tokenize(text)
+        return " ".join(tokens)
+    # English / non-CJK: return as-is, FTS5 unicode61 handles tokenization
+    return text
+
+
+# FTS5 operators that should NOT be tokenized
+_FTS5_OPERATORS = {"AND", "OR", "NOT", "NEAR"}
+
+# Characters that should be stripped from FTS5 query tokens.
+# Keep only word characters (letters, digits, underscore) and CJK ranges.
+_RE_FTS5_SPECIAL = re.compile(r'[^\w\u4e00-\u9fff\u3400-\u4dbf]')
+
+
+def _tokenize_fts_expression(expr: str) -> str:
+    """Tokenize terms in an FTS5 expression while preserving operators.
+
+    Raw FTS5 expressions like ``"machine AND learning"`` must have their
+    terms tokenized to match the indexed content, but FTS5
+    operators (AND, OR, NOT, NEAR) must remain untouched.
+
+    Handles:
+    - FTS5 keyword operators: AND, OR, NOT, NEAR
+    - Unary NOT prefix: ``-term`` → ``-tokenized``
+    - Quoted phrases: ``"exact phrase"`` → ``"tokenized phrase"``
+
+    Only applies jieba segmentation when CJK characters are detected.
+    """
+    # Split while preserving quoted segments as single units
+    tokens: list[str] = []
+    i = 0
+    while i < len(expr):
+        if expr[i] == '"':
+            end = expr.find('"', i + 1)
+            if end == -1:
+                end = len(expr)
+            tokens.append(expr[i:end + 1])
+            i = end + 1
+        elif expr[i].isspace():
+            i += 1
+        else:
+            end = i
+            while end < len(expr) and not expr[end].isspace() and expr[end] != '"':
+                end += 1
+            tokens.append(expr[i:end])
+            i = end
+
+    result = []
+    for part in tokens:
+        upper = part.upper()
+        if upper in _FTS5_OPERATORS:
+            result.append(upper)
+        elif part.startswith('"') and part.endswith('"'):
+            # Phrase query: "exact phrase" → tokenize inner content, keep quotes
+            inner = part[1:-1]
+            tokenized = _tokenize_for_fts(inner)
+            if tokenized.strip():
+                result.append(f'"{tokenized.strip()}"')
+        elif part.startswith('-'):
+            # Unary NOT: -term → -tokenized
+            tokenized = _tokenize_for_fts(part[1:])
+            if tokenized.strip():
+                result.append(f"-{tokenized.strip()}")
+        else:
+            tokenized = _tokenize_for_fts(part)
+            if tokenized.strip():
+                result.append(tokenized.strip())
+    return " ".join(result)
+
+
+# ---------------------------------------------------------------------------
+# FTS5 Index Engine
+# ---------------------------------------------------------------------------
+
+class FTS5Index:
+    """SQLite FTS5 full-text search index for tree-structured documents.
+
+    Features:
+      - WAL journal mode for concurrent read/write
+      - MD-aware columns: title, summary, body, code_blocks, front_matter
+      - Hierarchical column weighting via bm25() rank function
+      - Deferred indexing: call index_document() when ready
+      - Implements PreFilter protocol: score_nodes(query, doc_id)
+      - Supports FTS5 query syntax: AND, OR, NOT, NEAR, phrase "..."
+
+    In-memory mode (``db_path=None``):
+      All indexes are kept in SQLite ``:memory:`` — no file is written to disk.
+      Performance is excellent even with thousands of documents (5,000 docs < 10ms).
+      Indexes are lost when the process exits or the instance is closed.
+    """
+
+    def __init__(
+        self,
+        db_path: Optional[str] = None,
+        weights: Optional[dict] = None,
+        tokenize_log_path: Optional[str] = None,
+    ):
+        """
+        Args:
+            db_path: Path to SQLite database file. ``None`` for in-memory mode
+                (no file written to disk). Default: ``None``.
+            weights: column weight overrides for bm25() ranking.
+            tokenize_log_path: Optional path to write per-document tokenize logs.
+                When set, each call to ``index_document()`` appends deduplicated
+                token lists to this file.
+        """
+        self._db_path = db_path or ":memory:"
+        self._weights = {**_DEFAULT_WEIGHTS, **(weights or {})}
+        self._conn: Optional[sqlite3.Connection] = None
+        # Populated by index_document(); inspected by build_index for stats.
+        self.last_node_diff: dict[str, int] = {"added": 0, "changed": 0, "removed": 0, "kept": 0}
+        self._tokenize_log_path = tokenize_log_path
+        self._tokenize_log_file = None  # lazy-opened in _log_tokenize()
+        try:
+            self._init_db()
+        except Exception:
+            self.close()
+            raise
+
+    def _init_db(self) -> None:
+        """Initialize SQLite database with FTS5 virtual table (or fallback plain table)."""
+        if self._db_path != ":memory:":
+            os.makedirs(os.path.dirname(os.path.abspath(self._db_path)), exist_ok=True)
+
+        self._conn = sqlite3.connect(self._db_path, check_same_thread=False)
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA synchronous=NORMAL")
+
+        # Register REGEXP function for like_search(use_regex=True)
+        self._conn.create_function("REGEXP", 2, _sqlite_regexp)
+
+        self._use_fts5 = _check_fts5()
+
+        # Metadata table for nodes (structured fields for filtering)
+        self._conn.execute("""
+            CREATE TABLE IF NOT EXISTS nodes (
+                node_id TEXT NOT NULL,
+                doc_id TEXT NOT NULL,
+                title TEXT DEFAULT '',
+                summary TEXT DEFAULT '',
+                depth INTEGER DEFAULT 0,
+                line_start INTEGER,
+                line_end INTEGER,
+                parent_node_id TEXT,
+                content_hash TEXT,
+                PRIMARY KEY (doc_id, node_id)
+            )
+        """)
+
+        if self._use_fts5:
+            # FTS5 virtual table with content sync
+            # tokenize='unicode61' handles basic multi-language, but we pre-tokenize
+            # Chinese text with jieba and store space-separated tokens
+            self._conn.execute("""
+                CREATE VIRTUAL TABLE IF NOT EXISTS fts_nodes USING fts5(
+                    node_id UNINDEXED,
+                    doc_id UNINDEXED,
+                    title,
+                    summary,
+                    body,
+                    code_blocks,
+                    front_matter,
+                    tokenize='unicode61 remove_diacritics 2'
+                )
+            """)
+        else:
+            # Fallback: plain table with same columns for LIKE-based search
+            self._conn.execute("""
+                CREATE TABLE IF NOT EXISTS fts_nodes (
+                    rowid INTEGER PRIMARY KEY AUTOINCREMENT,
+                    node_id TEXT NOT NULL,
+                    doc_id TEXT NOT NULL,
+                    title TEXT DEFAULT '',
+                    summary TEXT DEFAULT '',
+                    body TEXT DEFAULT '',
+                    code_blocks TEXT DEFAULT '',
+                    front_matter TEXT DEFAULT ''
+                )
+            """)
+            self._conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_fts_nodes_doc_id ON fts_nodes (doc_id)"
+            )
+
+        # Document metadata table (also stores tree structure for persistence)
+        self._conn.execute("""
+            CREATE TABLE IF NOT EXISTS documents (
+                doc_id TEXT PRIMARY KEY,
+                doc_name TEXT DEFAULT '',
+                doc_description TEXT DEFAULT '',
+                source_path TEXT DEFAULT '',
+                source_type TEXT DEFAULT '',
+                structure_json TEXT DEFAULT '',
+                node_count INTEGER DEFAULT 0,
+                index_hash TEXT
+            )
+        """)
+
+        # Incremental index metadata (replaces _index_meta.json)
+        self._conn.execute("""
+            CREATE TABLE IF NOT EXISTS index_meta (
+                source_path TEXT PRIMARY KEY,
+                file_hash TEXT NOT NULL
+            )
+        """)
+
+        # Failed files tracking (auto-skip after consecutive parse failures)
+        self._conn.execute("""
+            CREATE TABLE IF NOT EXISTS failed_files (
+                source_path TEXT PRIMARY KEY,
+                fail_count INTEGER NOT NULL DEFAULT 1,
+                last_error TEXT DEFAULT '',
+                last_fail_time REAL NOT NULL,
+                file_hash TEXT DEFAULT ''
+            )
+        """)
+
+        # Vision parse queue (image files: placeholder first, background vision
+        # worker consumes serially and replaces the placeholder in-place).
+        # status: pending | processing | done | failed
+        self._conn.execute("""
+            CREATE TABLE IF NOT EXISTS vision_queue (
+                source_path TEXT PRIMARY KEY,
+                rel_path TEXT DEFAULT '',
+                status TEXT NOT NULL DEFAULT 'pending',
+                attempts INTEGER NOT NULL DEFAULT 0,
+                model TEXT DEFAULT '',
+                last_error TEXT DEFAULT '',
+                updated_at REAL NOT NULL
+            )
+        """)
+
+        # PST 邮件元数据（ADR-0005）：每封派生邮件文档一行，供物理 PST 的
+        # 邮件列表分页查询（主题/发件人/日期/文件夹 + 附件下载清单）。
+        self._conn.execute("""
+            CREATE TABLE IF NOT EXISTS pst_email_meta (
+                doc_id TEXT PRIMARY KEY,
+                pst_path TEXT NOT NULL,
+                entry_id TEXT NOT NULL,
+                subject TEXT DEFAULT '',
+                sender TEXT DEFAULT '',
+                date TEXT DEFAULT '',
+                folder TEXT DEFAULT '',
+                attachments_json TEXT DEFAULT '[]'
+            )
+        """)
+        self._conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_pst_email_meta_pst ON pst_email_meta (pst_path)"
+        )
+
+        # Performance indexes for large-scale document sets (10k+ docs)
+        self._conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_nodes_doc_id ON nodes (doc_id)"
+        )
+        self._conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_documents_source_path ON documents (source_path)"
+        )
+
+        self._conn.commit()
+
+    @property
+    def db_path(self) -> str:
+        return self._db_path
+
+    def close(self) -> None:
+        """Close database connection and tokenize log file."""
+        if self._tokenize_log_file:
+            self._tokenize_log_file.close()
+            self._tokenize_log_file = None
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+
+    def _log_tokenize(self, doc_id: str, tokens: set[str]) -> None:
+        """Append deduplicated token list for *doc_id* to the tokenize log."""
+        if not self._tokenize_log_path:
+            return
+        if self._tokenize_log_file is None:
+            self._tokenize_log_file = open(
+                self._tokenize_log_path, 'a', encoding='utf-8',
+            )
+        sorted_tokens = sorted(tokens)
+        self._tokenize_log_file.write(
+            f"--- {doc_id} ({len(sorted_tokens)} tokens) ---\n"
+        )
+        self._tokenize_log_file.write(" ".join(sorted_tokens) + "\n\n")
+        self._tokenize_log_file.flush()
+
+    def __del__(self):
+        try:
+            self.close()
+        except Exception:
+            pass
+
+    # -------------------------------------------------------------------
+    # Failed files tracking
+    # -------------------------------------------------------------------
+
+    def get_all_failed_files(self) -> dict[str, tuple[int, str, str]]:
+        """Batch load all failed file records.
+
+        Returns:
+            ``{source_path: (fail_count, file_hash, last_error)}``
+        """
+        rows = self._conn.execute(
+            "SELECT source_path, fail_count, file_hash, last_error FROM failed_files"
+        ).fetchall()
+        return {row[0]: (row[1], row[2], row[3] or "") for row in rows}
+
+    def upsert_failed_file(self, path: str, error_msg: str, file_hash: str = "") -> None:
+        """Insert or increment fail count for a file that failed to parse."""
+        import time as _time
+        now = _time.time()
+        self._conn.execute(
+            """INSERT INTO failed_files (source_path, fail_count, last_error, last_fail_time, file_hash)
+               VALUES (?, 1, ?, ?, ?)
+               ON CONFLICT(source_path) DO UPDATE SET
+                   fail_count = fail_count + 1,
+                   last_error = excluded.last_error,
+                   last_fail_time = excluded.last_fail_time,
+                   file_hash = excluded.file_hash
+            """,
+            (path, error_msg, now, file_hash),
+        )
+
+    def clear_failed_file(self, path: str) -> None:
+        """Remove a file's failed record after it is successfully indexed."""
+        self._conn.execute(
+            "DELETE FROM failed_files WHERE source_path = ?", (path,)
+        )
+
+    def clear_all_failed_files(self) -> None:
+        """Remove all failed file records (used during full rebuild)."""
+        with self._conn:
+            self._conn.execute("DELETE FROM failed_files")
+
+    def get_failed_files_summary(self) -> list[tuple[str, int, str]]:
+        """Return failed files list for display.
+
+        Returns:
+            list of ``(source_path, fail_count, last_error)``
+        """
+        rows = self._conn.execute(
+            "SELECT source_path, fail_count, last_error FROM failed_files ORDER BY fail_count DESC"
+        ).fetchall()
+        return [(row[0], row[1], row[2]) for row in rows]
+
+    # -------------------------------------------------------------------
+    # Vision parse queue (image files; consumed by host-side vision worker)
+    # -------------------------------------------------------------------
+
+    def vision_enqueue(self, source_path: str, rel_path: str = "") -> None:
+        """登记一个待视觉解析的图像文件（重复登记=文件变更，重置为 pending）。"""
+        import time as _time
+        self._conn.execute(
+            """INSERT INTO vision_queue (source_path, rel_path, status, attempts, model, last_error, updated_at)
+               VALUES (?, ?, 'pending', 0, '', '', ?)
+               ON CONFLICT(source_path) DO UPDATE SET
+                   rel_path = excluded.rel_path,
+                   status = 'pending',
+                   attempts = 0,
+                   last_error = '',
+                   updated_at = excluded.updated_at
+            """,
+            (source_path, rel_path, _time.time()),
+        )
+
+    def vision_next_pending(self) -> Optional[dict]:
+        """取出下一个待解析项并置为 processing（原子抢占）。无则返回 None。"""
+        import time as _time
+        row = self._conn.execute(
+            "SELECT source_path, rel_path, attempts FROM vision_queue "
+            "WHERE status = 'pending' ORDER BY updated_at LIMIT 1"
+        ).fetchone()
+        if row is None:
+            return None
+        cur = self._conn.execute(
+            "UPDATE vision_queue SET status = 'processing', updated_at = ? "
+            "WHERE source_path = ? AND status = 'pending'",
+            (_time.time(), row[0]),
+        )
+        self._conn.commit()
+        if cur.rowcount == 0:
+            return None  # 被其他消费者抢走
+        return {"source_path": row[0], "rel_path": row[1], "attempts": row[2]}
+
+    def vision_mark_done(self, source_path: str, model: str) -> bool:
+        """标记解析完成。仅当仍处于 processing 时生效（防止与 force 重建竞态）。"""
+        import time as _time
+        cur = self._conn.execute(
+            "UPDATE vision_queue SET status = 'done', model = ?, last_error = '', updated_at = ? "
+            "WHERE source_path = ? AND status = 'processing'",
+            (model, _time.time(), source_path),
+        )
+        self._conn.commit()
+        return cur.rowcount > 0
+
+    def vision_mark_failed(self, source_path: str, error: str, *, final: bool) -> None:
+        """记录一次失败；final=True（达到连败上限）时置 failed，否则回 pending 待重试。"""
+        import time as _time
+        self._conn.execute(
+            "UPDATE vision_queue SET status = ?, attempts = attempts + 1, last_error = ?, updated_at = ? "
+            "WHERE source_path = ?",
+            ("failed" if final else "pending", error[:500], _time.time(), source_path),
+        )
+        self._conn.commit()
+
+    def vision_remove(self, source_path: str) -> None:
+        """从队列移除（源文件被 prune / 删除时调用）。"""
+        self._conn.execute(
+            "DELETE FROM vision_queue WHERE source_path = ?", (source_path,)
+        )
+
+    def vision_clear(self) -> None:
+        """清空队列（force 全量重建时调用；重建过程会重新登记）。"""
+        with self._conn:
+            self._conn.execute("DELETE FROM vision_queue")
+
+    def vision_reset_stale_processing(self) -> int:
+        """启动时把残留的 processing（上次进程崩溃）重置回 pending。"""
+        with self._conn:
+            cur = self._conn.execute(
+                "UPDATE vision_queue SET status = 'pending' WHERE status = 'processing'"
+            )
+        return cur.rowcount
+
+    def vision_requeue_model_changed(self, current_tag: str) -> int:
+        """模型/prompt/协议版本变化时，把已完成或失败的项重新置为 pending 以便重解析。
+
+        失败项也纳入：改协议（如 OpenAI-compat → anthropic）后，旧协议下失败的
+        图像应在启动时自动重试，而不是永远卡在 failed。
+        """
+        import time as _time
+        with self._conn:
+            cur = self._conn.execute(
+                "UPDATE vision_queue SET status = 'pending', attempts = 0, last_error = '', updated_at = ? "
+                "WHERE status IN ('done', 'failed') AND model != ? AND model != 'manual'",
+                (_time.time(), current_tag),
+            )
+        return cur.rowcount
+
+    def vision_counts(self) -> dict[str, int]:
+        """各状态计数，供状态展示。如 ``{"pending": 3, "done": 12}``。"""
+        rows = self._conn.execute(
+            "SELECT status, COUNT(*) FROM vision_queue GROUP BY status"
+        ).fetchall()
+        return {row[0]: row[1] for row in rows}
+
+    # -------------------------------------------------------------------
+    # Indexing (Producer side)
+    # -------------------------------------------------------------------
+
+    def index_document(self, document, force: bool = False, auto_commit: bool = True,
+                       file_hash: Optional[str] = None) -> int:
+        """Index all nodes from a Document into FTS5.
+
+        Performs **node-level incremental diff**: only nodes whose content
+        actually changed (or appeared/disappeared) are removed/inserted into
+        the FTS5 index. The full document tree is still re-serialized into the
+        ``documents`` table so search continues to read the latest structure.
+
+        Stable ``node_id``s (assigned by :func:`treesearch.tree.assign_node_ids`)
+        are required for correct diffing — same logical position must yield the
+        same id across runs.
+
+        Side-effects on ``last_node_diff`` so callers (e.g. build_index) can
+        report diff stats.
+
+        Args:
+            document: Document object with structure tree.
+            force: re-index every node even if hashes match.
+            auto_commit: if False, skip commit (caller is responsible).
+            file_hash: optional file fingerprint to write to ``index_meta``
+                in the same transaction (avoids the "FTS written but
+                fingerprint missing → next run rebuilds" failure mode).
+
+        Returns:
+            number of nodes (re-)indexed in this call.
+        """
+        # Compute content hash for incremental check
+        content_str = json.dumps(document.structure, ensure_ascii=False, sort_keys=True)
+        content_hash = hashlib.md5(content_str.encode()).hexdigest()
+
+        # Check if already indexed (whole-document fast-path)
+        if not force:
+            row = self._conn.execute(
+                "SELECT index_hash FROM documents WHERE doc_id = ?",
+                (document.doc_id,),
+            ).fetchone()
+            if row and row[0] == content_hash:
+                # logger.debug("Document %s already indexed (hash match), skipping", document.doc_id)
+                self.last_node_diff = {"added": 0, "changed": 0, "removed": 0, "kept": 0}
+                # Still write index_meta if requested — we may have been called
+                # because the file fingerprint changed but the structure didn't.
+                if file_hash and document.metadata.get("source_path"):
+                    self._conn.execute(
+                        "INSERT OR REPLACE INTO index_meta (source_path, file_hash) VALUES (?, ?)",
+                        (document.metadata["source_path"], file_hash),
+                    )
+                    if auto_commit:
+                        self._conn.commit()
+                return 0
+
+        # ---- Compute node-level diff ----
+        from .tree import flatten_tree, build_tree_maps
+        _, parent_map, depth_map = build_tree_maps(document.structure)
+
+        all_nodes = [n for n in flatten_tree(document.structure) if n.get("node_id")]
+
+        new_hashes: dict[str, str] = {}
+        for node in all_nodes:
+            text = node.get("text", "")
+            new_hashes[node["node_id"]] = hashlib.md5(text.encode()).hexdigest()[:16]
+
+        if force:
+            old_hashes: dict[str, str] = {}
+        else:
+            old_hashes = {
+                nid: chash for (nid, chash) in self._conn.execute(
+                    "SELECT node_id, content_hash FROM nodes WHERE doc_id = ?",
+                    (document.doc_id,),
+                ).fetchall()
+            }
+
+        new_ids = set(new_hashes.keys())
+        old_ids = set(old_hashes.keys())
+        added = new_ids - old_ids
+        removed = old_ids - new_ids
+        changed = {nid for nid in (new_ids & old_ids) if new_hashes[nid] != old_hashes[nid]}
+        kept = (new_ids & old_ids) - changed
+
+        to_write = added | changed
+        diff_stats = {
+            "added": len(added),
+            "changed": len(changed),
+            "removed": len(removed),
+            "kept": len(kept),
+        }
+        self.last_node_diff = diff_stats
+
+        # ---- Stage all rows ahead of the transaction ----
+        node_rows: list[tuple] = []
+        fts_rows: list[tuple] = []
+        doc_tokens: set[str] = set()
+        for node in all_nodes:
+            nid = node["node_id"]
+            if nid not in to_write:
+                continue
+            title = node.get("title", "")
+            summary = node.get("summary", node.get("prefix_summary", ""))
+            text = node.get("text", "")
+            depth = depth_map.get(nid, 0)
+            node_rows.append((
+                nid, document.doc_id, title, summary, depth,
+                node.get("line_start"), node.get("line_end"),
+                parent_map.get(nid), new_hashes[nid],
+            ))
+
+            parsed = parse_md_node_text(text)
+            tok_title = _tokenize_for_fts(title)
+            tok_summary = _tokenize_for_fts(summary)
+            tok_body = _tokenize_for_fts(parsed["body"])
+            tok_code = _tokenize_for_fts(parsed["code_blocks"])
+            tok_fm = _tokenize_for_fts(parsed["front_matter"])
+
+            # 将文件名和路径分词后注入 front_matter，使文件名/路径可被 FTS5 搜索到
+            tok_doc_name = _tokenize_for_fts(document.doc_name)
+            if tok_doc_name and tok_doc_name not in tok_fm:
+                tok_fm = tok_doc_name + " " + tok_fm
+            source_path = document.metadata.get("source_path", "")
+            tok_path = _tokenize_for_fts(source_path)
+            if tok_path and tok_path not in tok_fm:
+                tok_fm = tok_path + " " + tok_fm
+            fts_rows.append((
+                nid, document.doc_id,
+                tok_title, tok_summary, tok_body, tok_code, tok_fm,
+            ))
+            for tok_str in (tok_title, tok_summary, tok_body, tok_code, tok_fm):
+                doc_tokens.update(tok_str.split())
+
+        self._log_tokenize(document.doc_id, doc_tokens)
+
+        # ---- Single atomic transaction ----
+        # Wraps deletes + inserts + document metadata + index_meta so a crash
+        # in the middle either rolls back fully or commits everything.
+        structure_json = json.dumps(document.structure, ensure_ascii=False)
+
+        if auto_commit:
+            txn_ctx = self._conn  # 'with conn' = implicit transaction
+        else:
+            txn_ctx = _NullContext()
+
+        with txn_ctx:
+            # Targeted deletes for removed + changed (NOT a wholesale wipe).
+            del_ids = removed | changed | (added if force else set())
+            if del_ids:
+                if self._use_fts5:
+                    placeholders = ",".join("?" for _ in del_ids)
+                    old_rowids = self._conn.execute(
+                        f"SELECT rowid FROM fts_nodes WHERE doc_id = ? AND node_id IN ({placeholders})",
+                        (document.doc_id, *del_ids),
+                    ).fetchall()
+                    if old_rowids:
+                        ph2 = ",".join("?" for _ in old_rowids)
+                        self._conn.execute(
+                            f"DELETE FROM fts_nodes WHERE rowid IN ({ph2})",
+                            [r[0] for r in old_rowids],
+                        )
+                else:
+                    placeholders = ",".join("?" for _ in del_ids)
+                    self._conn.execute(
+                        f"DELETE FROM fts_nodes WHERE doc_id = ? AND node_id IN ({placeholders})",
+                        (document.doc_id, *del_ids),
+                    )
+                placeholders = ",".join("?" for _ in del_ids)
+                self._conn.execute(
+                    f"DELETE FROM nodes WHERE doc_id = ? AND node_id IN ({placeholders})",
+                    (document.doc_id, *del_ids),
+                )
+
+            if node_rows:
+                self._conn.executemany(
+                    """INSERT INTO nodes
+                       (node_id, doc_id, title, summary, depth, line_start, line_end, parent_node_id, content_hash)
+                       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                    node_rows,
+                )
+            if fts_rows:
+                self._conn.executemany(
+                    """INSERT INTO fts_nodes
+                       (node_id, doc_id, title, summary, body, code_blocks, front_matter)
+                       VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                    fts_rows,
+                )
+
+            self._conn.execute(
+                """INSERT OR REPLACE INTO documents
+                   (doc_id, doc_name, doc_description, source_path, source_type,
+                    structure_json, node_count, index_hash)
+                   VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    document.doc_id, document.doc_name, document.doc_description,
+                    document.metadata.get("source_path", ""),
+                    document.source_type,
+                    structure_json,
+                    len(all_nodes), content_hash,
+                ),
+            )
+
+            if file_hash and document.metadata.get("source_path"):
+                self._conn.execute(
+                    "INSERT OR REPLACE INTO index_meta (source_path, file_hash) VALUES (?, ?)",
+                    (document.metadata["source_path"], file_hash),
+                )
+
+        logger.debug(
+            "FTS5 reindexed %s: +%d ~%d -%d (kept %d)",
+            document.doc_id, diff_stats["added"], diff_stats["changed"],
+            diff_stats["removed"], diff_stats["kept"],
+        )
+        return len(node_rows)
+
+    def commit(self) -> None:
+        """Manually commit pending changes to the database."""
+        self._conn.commit()
+
+    def index_documents(self, documents: list, force: bool = False) -> int:
+        """Batch index multiple documents.
+
+        Returns:
+            total number of nodes indexed
+        """
+        total = 0
+        for doc in documents:
+            total += self.index_document(doc, force=force)
+        return total
+
+    # -------------------------------------------------------------------
+    # Search (Consumer side)
+    # -------------------------------------------------------------------
+
+    def _build_match_expr(self, query: str, fts_expression: Optional[str] = None) -> Optional[str]:
+        """Build FTS5 MATCH expression from query (cached tokenization).
+
+        Returns None if no valid tokens could be extracted.
+        """
+        if fts_expression:
+            return _tokenize_fts_expression(fts_expression)
+
+        tokens = _tokenize_for_fts(query)
+        if not tokens.strip():
+            return None
+        words = tokens.split()
+        clean_words = []
+        for w in words:
+            cleaned = _RE_FTS5_SPECIAL.sub("", w).strip()
+            if cleaned and cleaned.upper() not in _FTS5_OPERATORS:
+                clean_words.append(cleaned)
+        if not clean_words:
+            return None
+        if len(clean_words) > 1:
+            return " OR ".join(clean_words)
+        return clean_words[0]
+
+    def search(
+        self,
+        query: str,
+        doc_id: Optional[str] = None,
+        top_k: int = 20,
+        fts_expression: Optional[str] = None,
+        _precomputed_match_expr: Optional[str] = None,
+    ) -> list[dict]:
+        """Search nodes using FTS5 BM25 ranking (or LIKE fallback).
+
+        Args:
+            query: natural language query (will be tokenized)
+            doc_id: optional filter by document
+            top_k: max results
+            fts_expression: raw FTS5 query expression (overrides query tokenization).
+                            Supports AND, OR, NOT, NEAR, phrases.
+            _precomputed_match_expr: internal — skip tokenization if already computed.
+
+        Returns:
+            list of {node_id, doc_id, title, summary, fts_score, depth}
+        """
+        if not self._use_fts5:
+            return self._search_like(query, doc_id=doc_id, top_k=top_k)
+
+        if _precomputed_match_expr is not None:
+            match_expr = _precomputed_match_expr
+        else:
+            match_expr = self._build_match_expr(query, fts_expression)
+            if match_expr is None:
+                return []
+
+        # Phase 1: phrase boosting for multi-word queries
+        # Run a separate phrase match query and record which nodes get a boost
+        phrase_boost_nids: set[str] = set()
+        if not fts_expression and _precomputed_match_expr is None and len(query.split()) >= 2:
+            # Build phrase expression from original (unstemmed) query words
+            raw_words = [w.lower().strip() for w in re.split(r'\W+', query) if w.strip() and len(w.strip()) > 2]
+            if len(raw_words) >= 2:
+                # Try phrase match: "word1 word2 ..."
+                phrase_expr = '"' + ' '.join(raw_words) + '"'
+                try:
+                    if doc_id:
+                        phrase_rows = self._conn.execute(
+                            f"SELECT f.node_id FROM fts_nodes f WHERE fts_nodes MATCH ? AND f.doc_id = ? LIMIT 50",
+                            (phrase_expr, doc_id),
+                        ).fetchall()
+                    else:
+                        phrase_rows = self._conn.execute(
+                            f"SELECT f.node_id FROM fts_nodes f WHERE fts_nodes MATCH ? LIMIT 50",
+                            (phrase_expr,),
+                        ).fetchall()
+                    phrase_boost_nids = {r[0] for r in phrase_rows}
+                except sqlite3.OperationalError:
+                    pass  # phrase query syntax error, skip boost
+
+        # Build SQL with column weights for bm25()
+        # bm25(fts_nodes, w1, w2, w3, w4, w5) where weights correspond to:
+        # node_id(UNINDEXED), doc_id(UNINDEXED), title, summary, body, code_blocks, front_matter
+        w = self._weights
+        weight_args = f"{w['title']}, {w['summary']}, {w['body']}, {w['code_blocks']}, {w['front_matter']}"
+
+        # Query FTS5 directly without JOIN to nodes table, because fts_nodes
+        # stores chunk node_ids (e.g. "0_chunk0") that don't match the original
+        # node_ids in nodes table (e.g. "0"). Metadata is looked up separately.
+        if doc_id:
+            sql = f"""
+                SELECT f.node_id, f.doc_id, f.title, f.summary,
+                       bm25(fts_nodes, {weight_args}) AS rank_score
+                FROM fts_nodes f
+                WHERE fts_nodes MATCH ?
+                  AND f.doc_id = ?
+                ORDER BY rank_score
+                LIMIT ?
+            """
+            params = (match_expr, doc_id, top_k)
+        else:
+            sql = f"""
+                SELECT f.node_id, f.doc_id, f.title, f.summary,
+                       bm25(fts_nodes, {weight_args}) AS rank_score
+                FROM fts_nodes f
+                WHERE fts_nodes MATCH ?
+                ORDER BY rank_score
+                LIMIT ?
+            """
+            params = (match_expr, top_k)
+
+        try:
+            rows = self._conn.execute(sql, params).fetchall()
+        except sqlite3.OperationalError as e:
+            logger.warning("FTS5 query error: %s, query=%r", e, match_expr)
+            rows = []
+
+        # Pre-fetch node metadata (depth, title, summary) for deduped node_ids
+        unique_nids_in_result = {r[0] for r in rows}
+        node_meta: dict[tuple[str, str], dict] = {}
+        if unique_nids_in_result:
+            # Batch lookup from nodes table
+            for raw_nid in unique_nids_in_result:
+                for r in rows:
+                    if r[0] == raw_nid:
+                        did = r[1]
+                        break
+                else:
+                    continue
+                meta_row = self._conn.execute(
+                    "SELECT title, summary, depth FROM nodes WHERE node_id = ? AND doc_id = ?",
+                    (raw_nid, did),
+                ).fetchone()
+                if meta_row:
+                    node_meta[(raw_nid, did)] = {"title": meta_row[0], "summary": meta_row[1], "depth": meta_row[2]}
+
+        results = []
+        seen_nids: dict[str, int] = {}  # track dedup by node_id
+        for row in rows:
+            # bm25() returns negative values (lower = more relevant)
+            fts_score = -row[4] if row[4] else 0.0
+            # Apply phrase boost: nodes matching exact phrase get 50% score bonus
+            if row[0] in phrase_boost_nids:
+                fts_score *= 1.5
+            nid = row[0]
+            if nid in seen_nids:
+                # Keep the higher score for the same node
+                idx = seen_nids[nid]
+                if fts_score > results[idx]["fts_score"]:
+                    results[idx]["fts_score"] = round(fts_score, 6)
+                continue
+            seen_nids[nid] = len(results)
+            meta = node_meta.get((nid, row[1]))
+            results.append({
+                "node_id": nid,
+                "doc_id": row[1],
+                "title": meta["title"] if meta else row[2],
+                "summary": meta["summary"] if meta else row[3],
+                "depth": meta["depth"] if meta else 0,
+                "fts_score": round(fts_score, 6),
+            })
+
+        # Re-sort after phrase boosting
+        if phrase_boost_nids:
+            results.sort(key=lambda x: -x["fts_score"])
+
+        return results
+
+    def _search_like(
+        self,
+        query: str,
+        doc_id: Optional[str] = None,
+        top_k: int = 20,
+    ) -> list[dict]:
+        """Fallback search using LIKE when FTS5 is unavailable.
+
+        Splits query into keywords and scores nodes by weighted keyword hits
+        across (title, summary, body, code_blocks, front_matter).
+        """
+        tokens = _tokenize_for_fts(query)
+        if not tokens.strip():
+            return []
+        keywords = [kw.strip().lower() for kw in tokens.split() if kw.strip()]
+        if not keywords:
+            return []
+
+        w = self._weights
+
+        # Pre-fetch all node metadata to avoid N+1 queries
+        if doc_id:
+            meta_rows = self._conn.execute(
+                "SELECT node_id, doc_id, title, summary, depth FROM nodes WHERE doc_id = ?",
+                (doc_id,),
+            ).fetchall()
+        else:
+            meta_rows = self._conn.execute(
+                "SELECT node_id, doc_id, title, summary, depth FROM nodes"
+            ).fetchall()
+        meta_map = {(r[0], r[1]): {"title": r[2], "summary": r[3], "depth": r[4]} for r in meta_rows}
+
+        if doc_id:
+            rows = self._conn.execute(
+                """SELECT node_id, doc_id, title, summary, body, code_blocks, front_matter
+                   FROM fts_nodes WHERE doc_id = ?""",
+                (doc_id,),
+            ).fetchall()
+        else:
+            rows = self._conn.execute(
+                "SELECT node_id, doc_id, title, summary, body, code_blocks, front_matter FROM fts_nodes"
+            ).fetchall()
+
+        scored: list[tuple[float, dict]] = []
+        seen_original_nids: dict[str, int] = {}
+        for row in rows:
+            nid, did, title, summary, body, code_blocks, front_matter = row
+            score = 0.0
+            fields = [
+                (title or "", w["title"]),
+                (summary or "", w["summary"]),
+                (body or "", w["body"]),
+                (code_blocks or "", w["code_blocks"]),
+                (front_matter or "", w["front_matter"]),
+            ]
+            for kw in keywords:
+                for text, weight in fields:
+                    if kw in text.lower():
+                        score += weight
+            if score > 0:
+                if nid in seen_original_nids:
+                    idx = seen_original_nids[nid]
+                    if score > scored[idx][0]:
+                        meta = meta_map.get((nid, did))
+                        scored[idx] = (score, {
+                            "node_id": nid,
+                            "doc_id": did,
+                            "title": meta["title"] if meta else title,
+                            "summary": meta["summary"] if meta else summary,
+                            "depth": meta["depth"] if meta else 0,
+                            "fts_score": round(score, 6),
+                        })
+                    continue
+                seen_original_nids[nid] = len(scored)
+                meta = meta_map.get((nid, did))
+                scored.append((score, {
+                    "node_id": nid,
+                    "doc_id": did,
+                    "title": meta["title"] if meta else title,
+                    "summary": meta["summary"] if meta else summary,
+                    "depth": meta["depth"] if meta else 0,
+                    "fts_score": round(score, 6),
+                }))
+
+        scored.sort(key=lambda x: -x[0])
+        return [item[1] for item in scored[:top_k]]
+
+    def like_search(self, query: str, top_k: int = 20, use_regex: bool = False) -> list[dict]:
+        """Substring or regex search on nodes.summary/title (original text, not tokenized).
+
+        Fallback when FTS5 tokenization causes query mismatches.
+        Searches the ``nodes`` table which stores original (un-tokenized) text,
+        so phrases that were split by jieba can still be matched.
+
+        Args:
+            query: search string or regex pattern (when use_regex=True).
+            top_k: max results.
+            use_regex: if True, treat query as a regular expression (SQLite REGEXP).
+
+        If ``nodes`` yields no matches, searches ``documents.structure_json``
+        which contains the full original document structure.
+
+        Returns same format as search(): list of {node_id, doc_id, title, summary, fts_score, depth}.
+        """
+        results: list[dict] = []
+        seen: set[str] = set()
+
+        # Build match expression based on mode
+        if use_regex:
+            compiled = re.compile(query, re.IGNORECASE)
+            where_clause = "n.summary REGEXP ? OR n.title REGEXP ?"
+            params = (query, query)
+        else:
+            pattern = f"%{query}%"
+            where_clause = "n.summary LIKE ? OR n.title LIKE ?"
+            params = (pattern, pattern)
+
+        # Phase 1: Search nodes.summary and nodes.title (original text)
+        rows = self._conn.execute(
+            f"""SELECT n.node_id, n.doc_id, n.title, n.summary, n.depth,
+                      d.doc_name, d.source_path
+               FROM nodes n
+               JOIN documents d ON n.doc_id = d.doc_id
+               WHERE {where_clause}
+               ORDER BY n.depth ASC""",
+            params,
+        ).fetchall()
+
+        for row in rows:
+            nid, did = row[0], row[1]
+            key = f"{did}/{nid}"
+            if key in seen:
+                continue
+            seen.add(key)
+            results.append({
+                "node_id": nid,
+                "doc_id": did,
+                "title": row[2] or "",
+                "summary": row[3] or "",
+                "depth": row[4] or 0,
+                "fts_score": 1.0,
+            })
+
+        if results:
+            return results[:top_k]
+
+        # Phase 2: Search documents.structure_json (full original text)
+        if use_regex:
+            match_fn = compiled.search
+        else:
+            query_lower = query.lower()
+            match_fn = lambda t: query_lower in t.lower() if t else False
+
+        try:
+            if use_regex:
+                doc_rows = self._conn.execute(
+                    "SELECT doc_id, structure_json FROM documents WHERE structure_json REGEXP ?",
+                    (query,),
+                ).fetchall()
+            else:
+                doc_rows = self._conn.execute(
+                    "SELECT doc_id, structure_json FROM documents WHERE structure_json LIKE ?",
+                    (pattern,),
+                ).fetchall()
+        except Exception:
+            return []
+
+        for doc_id, structure_json in doc_rows:
+            if not structure_json:
+                continue
+            try:
+                nodes_data = json.loads(structure_json)
+            except (json.JSONDecodeError, TypeError):
+                continue
+            if not isinstance(nodes_data, list):
+                continue
+            from .tree import flatten_tree
+            for node in flatten_tree(nodes_data):
+                text = node.get("text", "") or ""
+                title = node.get("title", "") or ""
+                if match_fn(text) or match_fn(title):
+                    nid = node.get("node_id", "")
+                    key = f"{doc_id}/{nid}"
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    # 提取包含匹配关键词的片段
+                    snippet = _extract_match_snippet(text, query, use_regex, size=300)
+                    results.append({
+                        "node_id": nid,
+                        "doc_id": doc_id,
+                        "title": title,
+                        "summary": snippet,
+                        "depth": node.get("depth", 0),
+                        "fts_score": 0.5,
+                    })
+
+        return results[:top_k]
+
+    def search_with_aggregation(
+        self,
+        query: str,
+        group_by_doc: bool = True,
+        top_k: int = 20,
+        fts_expression: Optional[str] = None,
+    ) -> list[dict]:
+        """Search with SQL aggregation capabilities.
+
+        Returns per-document aggregated results: total hits, max score, avg score.
+        """
+        if not group_by_doc:
+            return self.search(query, top_k=top_k, fts_expression=fts_expression)
+
+        # Two-step: first get all matched nodes with scores, then aggregate in Python
+        results = self.search(query, top_k=200, fts_expression=fts_expression)
+        if not results:
+            return []
+
+        doc_agg: dict[str, dict] = {}
+        for r in results:
+            did = r["doc_id"]
+            if did not in doc_agg:
+                # Fetch doc_name
+                row = self._conn.execute(
+                    "SELECT doc_name FROM documents WHERE doc_id = ?", (did,)
+                ).fetchone()
+                doc_agg[did] = {
+                    "doc_id": did,
+                    "doc_name": row[0] if row else "",
+                    "hit_count": 0,
+                    "best_score": 0.0,
+                    "total_score": 0.0,
+                }
+            doc_agg[did]["hit_count"] += 1
+            doc_agg[did]["best_score"] = max(doc_agg[did]["best_score"], r["fts_score"])
+            doc_agg[did]["total_score"] += r["fts_score"]
+
+        agg_results = []
+        for agg in doc_agg.values():
+            agg["avg_score"] = round(agg["total_score"] / agg["hit_count"], 6)
+            agg["best_score"] = round(agg["best_score"], 6)
+            del agg["total_score"]
+            agg_results.append(agg)
+
+        agg_results.sort(key=lambda x: -x["best_score"])
+        return agg_results[:top_k]
+
+    def score_nodes(self, query: str, doc_id: str, ancestor_decay: float = 0.6) -> dict[str, float]:
+        """PreFilter protocol: return {node_id: score} for search() integration.
+
+        This allows FTS5Index to be used as a drop-in PreFilter in the search pipeline.
+
+        Includes:
+        - Ancestor score propagation: parent nodes inherit child scores
+
+        For scoring multiple documents at once, prefer score_nodes_batch() which uses
+        a single SQL query instead of one query per document.
+        """
+        result = self.score_nodes_batch(query, doc_ids=[doc_id], ancestor_decay=ancestor_decay)
+        return result.get(doc_id, {})
+
+    def score_nodes_batch(
+        self,
+        query: str,
+        doc_ids: list[str] | None = None,
+        ancestor_decay: float = 0.6,
+        fts_expression: Optional[str] = None,
+    ) -> dict[str, dict[str, float]]:
+        """Batch version of score_nodes: score all documents in a single SQL query.
+
+        Returns {doc_id: {node_id: score}} for all matched documents.
+
+        This is the fast path for tree search, replacing the N_docs loop:
+            # Before (slow): N SQL queries
+            for doc in documents:
+                scores = fts_index.score_nodes(query, doc.doc_id)
+
+            # After (fast): 1 SQL query
+            all_scores = fts_index.score_nodes_batch(query, [d.doc_id for d in documents])
+
+        Args:
+            query: natural language query (tokenized internally)
+            doc_ids: optional filter to specific documents
+            ancestor_decay: propagation weight from child scores to parent nodes (0 = off)
+            fts_expression: raw FTS5 MATCH expression that *overrides* automatic query
+                tokenization. Supports full FTS5 syntax including prefix matching (*),
+                AND/OR/NOT, NEAR(), and column filters.
+
+                Examples::
+
+                    # Prefix match: "fts" matches fts, fts5, ftsearch, ...
+                    fts_expression="fts*"
+
+                    # Multi-term prefix OR
+                    fts_expression="fts* OR python*"
+
+                    # Exact phrase
+                    fts_expression='"machine learning"'
+
+                    # Column-scoped search
+                    fts_expression="title : config*"
+
+                    # Build with helper
+                    expr = FTS5Index.build_fts_expression(["fts", "python"],
+                                                          prefix=True, operator="OR")
+        """
+        match_expr = self._build_match_expr(query, fts_expression)
+        if match_expr is None:
+            return {}
+
+        w = self._weights
+        weight_args = f"{w['title']}, {w['summary']}, {w['body']}, {w['code_blocks']}, {w['front_matter']}"
+
+        # Single SQL query across all requested doc_ids (or entire index)
+        if doc_ids:
+            placeholders = ",".join("?" * len(doc_ids))
+            sql = f"""
+                SELECT f.node_id, f.doc_id,
+                       bm25(fts_nodes, {weight_args}) AS rank_score
+                FROM fts_nodes f
+                WHERE fts_nodes MATCH ?
+                  AND f.doc_id IN ({placeholders})
+                ORDER BY rank_score
+                LIMIT 5000
+            """
+            params = (match_expr, *doc_ids)
+        else:
+            sql = f"""
+                SELECT f.node_id, f.doc_id,
+                       bm25(fts_nodes, {weight_args}) AS rank_score
+                FROM fts_nodes f
+                WHERE fts_nodes MATCH ?
+                ORDER BY rank_score
+                LIMIT 5000
+            """
+            params = (match_expr,)
+
+        try:
+            rows = self._conn.execute(sql, params).fetchall()
+        except Exception:
+            return {}
+
+        if not rows:
+            return {}
+
+        # Group raw scores by doc_id
+        per_doc_raw: dict[str, dict[str, float]] = {}
+        for node_id, doc_id, rank_score in rows:
+            fts_score = -rank_score if rank_score else 0.0
+            per_doc_raw.setdefault(doc_id, {})
+            old = per_doc_raw[doc_id].get(node_id, 0.0)
+            per_doc_raw[doc_id][node_id] = max(old, fts_score)
+
+        # Per-doc: normalize + ancestor propagation in one pass
+        result: dict[str, dict[str, float]] = {}
+        doc_children_map: dict[str, dict[str, list[str]]] = {}
+        if ancestor_decay > 0 and per_doc_raw:
+            # Single query to fetch parent maps for all affected docs
+            affected_docs = list(per_doc_raw.keys())
+            ph = ",".join("?" * len(affected_docs))
+            parent_rows = self._conn.execute(
+                f"SELECT doc_id, node_id, parent_node_id FROM nodes WHERE doc_id IN ({ph})",
+                affected_docs,
+            ).fetchall()
+            # Build per-doc children maps for bottom-up propagation
+            for d_id, nid, pid in parent_rows:
+                if pid:
+                    doc_children_map.setdefault(d_id, {}).setdefault(pid, []).append(nid)
+
+        for doc_id, raw_scores in per_doc_raw.items():
+            # Normalize to [0, 1]
+            max_s = max(raw_scores.values()) if raw_scores else 1.0
+            if max_s <= 0:
+                max_s = 1.0
+            scores = {nid: s / max_s for nid, s in raw_scores.items()}
+
+            # Ancestor propagation
+            if ancestor_decay > 0:
+                children_map = doc_children_map.get(doc_id, {})
+                for pid, cids in children_map.items():
+                    child_scores = [scores.get(c, 0.0) for c in cids]
+                    if not child_scores:
+                        continue
+                    bonus = ancestor_decay * max(child_scores)
+                    scores[pid] = scores.get(pid, 0.0) + bonus
+
+                # Re-normalize after propagation
+                final_max = max(scores.values()) if scores else 1.0
+                if final_max > 1.0:
+                    scores = {nid: s / final_max for nid, s in scores.items()}
+
+            result[doc_id] = {nid: round(s, 6) for nid, s in scores.items()}
+
+        return result
+
+    def ranked_node_ids(
+        self,
+        query: str,
+        doc_ids: list[str] | None = None,
+        top_k: int = 10,
+        fts_expression: Optional[str] = None,
+    ) -> list[str]:
+        """Convenience: return top-k node IDs ranked by FTS5 score.
+
+        Wraps score_nodes_batch → flatten → sort → top-k in one call.
+        Useful for flat FTS5 evaluation without manual dict wrangling.
+
+        Args:
+            query: search query
+            doc_ids: optional filter to specific documents
+            top_k: max results
+            fts_expression: raw FTS5 MATCH expression (overrides query tokenization).
+                Use for prefix matching, boolean operators, NEAR(), etc.
+                Example: ``fts_expression="config*"`` matches config, configuration, ...
+
+        Returns:
+            list of node_id strings, highest score first
+        """
+        batch = self.score_nodes_batch(query, doc_ids=doc_ids, fts_expression=fts_expression)
+        all_scored: list[tuple[str, float]] = []
+        for nscores in batch.values():
+            all_scored.extend(nscores.items())
+        all_scored.sort(key=lambda x: -x[1])
+        return [nid for nid, _ in all_scored[:top_k]]
+
+    # -------------------------------------------------------------------
+    # Document persistence (tree structure storage)
+    # -------------------------------------------------------------------
+
+    def save_document(self, document, auto_commit: bool = True) -> None:
+        """Save/update a Document's tree structure into the DB.
+
+        This persists the tree structure so that JSON files are no longer needed.
+        FTS indexing is NOT performed here — call index_document() separately.
+
+        Args:
+            document: Document object with structure tree
+            auto_commit: if False, skip commit (caller is responsible for committing)
+        """
+        from .tree import flatten_tree
+        structure_json = json.dumps(document.structure, ensure_ascii=False)
+        content_hash = hashlib.md5(structure_json.encode()).hexdigest()
+        self._conn.execute(
+            """INSERT OR REPLACE INTO documents
+               (doc_id, doc_name, doc_description, source_path, source_type, structure_json, node_count, index_hash)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+            (
+                document.doc_id, document.doc_name, document.doc_description,
+                document.metadata.get("source_path", ""),
+                document.source_type,
+                structure_json,
+                len(flatten_tree(document.structure)),
+                content_hash,
+            ),
+        )
+        if auto_commit:
+            self._conn.commit()
+
+    def load_document(self, doc_id: str):
+        """Load a single Document from the DB by doc_id.
+
+        Returns:
+            Document object, or None if not found.
+        """
+        from .tree import Document
+        row = self._conn.execute(
+            "SELECT doc_id, doc_name, doc_description, source_path, source_type, structure_json FROM documents WHERE doc_id = ?",
+            (doc_id,),
+        ).fetchone()
+        if not row:
+            return None
+        structure = json.loads(row[5]) if row[5] else []
+        return Document(
+            doc_id=row[0],
+            doc_name=row[1],
+            structure=structure,
+            doc_description=row[2] or "",
+            metadata={"source_path": row[3] or ""},
+            source_type=row[4] or "",
+        )
+
+    def load_document_by_source_path(self, source_path: str) -> Optional["Document"]:
+        """按 source_path 加载单个 Document（含 structure_json 反序列化）。
+
+        Args:
+            source_path: 文件绝对路径，必须与索引时存入 documents.source_path 完全一致。
+
+        Returns:
+            Document 对象；无匹配返回 None。
+        """
+        from .tree import Document  # 局部导入避免循环依赖
+
+        row = self._conn.execute(
+            "SELECT doc_id, doc_name, doc_description, source_path, source_type, structure_json "
+            "FROM documents WHERE source_path = ? LIMIT 1",
+            (source_path,),
+        ).fetchone()
+        if not row:
+            return None
+        doc_id, doc_name, doc_description, sp, source_type, structure_json = row
+        structure = json.loads(structure_json) if structure_json else []
+        return Document(
+            doc_id=doc_id,
+            doc_name=doc_name or "",
+            doc_description=doc_description or "",
+            structure=structure,
+            source_type=source_type or "",
+            metadata={"source_path": sp or ""},
+        )
+
+    def load_all_documents(self) -> list:
+        """Load all Documents stored in the DB.
+
+        Returns:
+            List of Document objects.
+        """
+        from .tree import Document
+        rows = self._conn.execute(
+            "SELECT doc_id, doc_name, doc_description, source_path, source_type, structure_json FROM documents ORDER BY doc_id"
+        ).fetchall()
+        documents = []
+        for row in rows:
+            structure = json.loads(row[5]) if row[5] else []
+            documents.append(Document(
+                doc_id=row[0],
+                doc_name=row[1],
+                structure=structure,
+                doc_description=row[2] or "",
+                metadata={"source_path": row[3] or ""},
+                source_type=row[4] or "",
+            ))
+        return documents
+
+    def delete_document(self, doc_id: str) -> bool:
+        """Delete a document and all its indexed data from the DB atomically.
+
+        Clears all four storage locations in a single transaction:
+        - ``fts_nodes`` (FTS5 inverted index entries)
+        - ``nodes`` (structured node metadata)
+        - ``documents`` (tree structure + document metadata)
+        - ``index_meta`` (incremental indexing fingerprint)
+
+        Clearing ``index_meta`` is critical: without it the incremental indexing
+        logic would see the file as already-processed and silently skip re-indexing
+        it after a subsequent ``index()`` call.
+
+        Args:
+            doc_id: document identifier to delete.
+
+        Returns:
+            ``True`` if the document existed and was deleted, ``False`` if it was
+            not found (operation is idempotent — no exception is raised).
+        """
+        # Check existence before entering the transaction so we can return a
+        # meaningful bool without relying on changes_count() across all tables.
+        row = self._conn.execute(
+            "SELECT source_path FROM documents WHERE doc_id = ?", (doc_id,)
+        ).fetchone()
+
+        if row is None:
+            logger.warning("delete_document: doc_id=%r not found, nothing deleted", doc_id)
+            return False
+
+        source_path = row[0] or ""
+
+        try:
+            # Single atomic transaction — all-or-nothing.
+            with self._conn:
+                # 1. FTS5 virtual table: must delete by rowid (UNINDEXED columns
+                #    cannot be used in a WHERE clause for DELETE on FTS5 tables).
+                if self._use_fts5:
+                    old_rowids = self._conn.execute(
+                        "SELECT rowid FROM fts_nodes WHERE doc_id = ?", (doc_id,)
+                    ).fetchall()
+                    if old_rowids:
+                        placeholders = ",".join("?" for _ in old_rowids)
+                        self._conn.execute(
+                            f"DELETE FROM fts_nodes WHERE rowid IN ({placeholders})",
+                            [r[0] for r in old_rowids],
+                        )
+                else:
+                    self._conn.execute("DELETE FROM fts_nodes WHERE doc_id = ?", (doc_id,))
+
+                # 2. Structured node metadata.
+                self._conn.execute("DELETE FROM nodes WHERE doc_id = ?", (doc_id,))
+
+                # 3. Document record (tree structure + metadata).
+                self._conn.execute("DELETE FROM documents WHERE doc_id = ?", (doc_id,))
+
+                # 4. Incremental index fingerprint — CRITICAL: must be cleared so
+                #    that the next index() call re-processes this file rather than
+                #    skipping it as "already indexed".
+                if source_path:
+                    self._conn.execute(
+                        "DELETE FROM index_meta WHERE source_path = ?", (source_path,)
+                    )
+
+        except sqlite3.DatabaseError as e:
+            logger.error(
+                "delete_document failed for doc_id=%r (source_path=%r): %s",
+                doc_id, source_path, e,
+            )
+            raise
+
+        logger.info("Deleted document doc_id=%r (source_path=%r)", doc_id, source_path)
+        return True
+
+    def remove_document(self, doc_id: str) -> None:
+        """Remove a document from the DB.
+
+        .. deprecated::
+            Use :meth:`delete_document` instead.  ``delete_document`` fixes a
+            bug where ``index_meta`` was not cleared, uses an atomic transaction,
+            and returns a boolean indicating whether the document existed.
+        """
+        self.delete_document(doc_id)
+
+    def find_doc_by_fingerprint(self, file_hash: str, exclude_paths: Optional[set] = None) -> Optional[str]:
+        """Find a doc whose ``index_meta.file_hash`` matches ``file_hash``.
+
+        Used by the indexer to detect file moves/renames: if the same fingerprint
+        already exists under a different source_path, we can update the path
+        instead of re-parsing+re-indexing the file.
+
+        Args:
+            file_hash: target fingerprint string.
+            exclude_paths: source_paths to skip (e.g. paths still on disk).
+
+        Returns:
+            doc_id of the matching document, or None if no candidate.
+        """
+        rows = self._conn.execute(
+            "SELECT source_path FROM index_meta WHERE file_hash = ?",
+            (file_hash,),
+        ).fetchall()
+        for (sp,) in rows:
+            if exclude_paths and sp in exclude_paths:
+                continue
+            doc_id = self.get_doc_id_by_source_path(sp)
+            if doc_id is not None:
+                return doc_id
+        return None
+
+    def update_source_path(self, doc_id: str, new_source_path: str) -> None:
+        """Atomically remap a document's source_path (move/rename support)."""
+        with self._conn:
+            old_row = self._conn.execute(
+                "SELECT source_path FROM documents WHERE doc_id = ?", (doc_id,)
+            ).fetchone()
+            self._conn.execute(
+                "UPDATE documents SET source_path = ? WHERE doc_id = ?",
+                (new_source_path, doc_id),
+            )
+            if old_row and old_row[0]:
+                self._conn.execute(
+                    "DELETE FROM index_meta WHERE source_path = ?", (old_row[0],)
+                )
+
+    def rename_document(
+        self,
+        old_doc_id: str,
+        new_doc_id: str,
+        new_doc_name: str,
+        new_source_path: str,
+    ) -> bool:
+        """Rename a doc in place across nodes / fts_nodes / documents / index_meta.
+
+        Used by `build_index`'s move-detection pre-pass when a file with an
+        unchanged content fingerprint shows up under a new path. Keeps the
+        doc identity (`doc_id` and `doc_name`) consistent with the new file
+        basename so callers don't see stale names after a rename.
+
+        Returns ``False`` (and writes nothing) when:
+          - the original ``old_doc_id`` no longer exists, or
+          - ``new_doc_id`` is already taken by a *different* document — in
+            that case the caller should fall back to a full re-index.
+        """
+        old_row = self._conn.execute(
+            "SELECT doc_id, source_path FROM documents WHERE doc_id = ?",
+            (old_doc_id,),
+        ).fetchone()
+        if old_row is None:
+            return False
+        old_source_path = old_row[1] or ""
+
+        if new_doc_id != old_doc_id:
+            clash = self._conn.execute(
+                "SELECT 1 FROM documents WHERE doc_id = ?", (new_doc_id,)
+            ).fetchone()
+            if clash is not None:
+                return False
+
+        with self._conn:
+            if new_doc_id != old_doc_id:
+                self._conn.execute(
+                    "UPDATE nodes SET doc_id = ? WHERE doc_id = ?",
+                    (new_doc_id, old_doc_id),
+                )
+                self._conn.execute(
+                    "UPDATE fts_nodes SET doc_id = ? WHERE doc_id = ?",
+                    (new_doc_id, old_doc_id),
+                )
+                self._conn.execute(
+                    "UPDATE documents SET doc_id = ?, doc_name = ?, source_path = ? "
+                    "WHERE doc_id = ?",
+                    (new_doc_id, new_doc_name, new_source_path, old_doc_id),
+                )
+            else:
+                self._conn.execute(
+                    "UPDATE documents SET doc_name = ?, source_path = ? WHERE doc_id = ?",
+                    (new_doc_name, new_source_path, old_doc_id),
+                )
+            if old_source_path and old_source_path != new_source_path:
+                self._conn.execute(
+                    "DELETE FROM index_meta WHERE source_path = ?", (old_source_path,)
+                )
+
+        return True
+
+    def delete_documents(self, doc_ids: list[str]) -> int:
+        """Batch-delete multiple documents in a single transaction.
+
+        Significantly faster than calling ``delete_document`` in a loop because
+        every table is hit once with an ``IN (...)`` clause.
+
+        Returns the number of documents that actually existed and were removed.
+        """
+        if not doc_ids:
+            return 0
+
+        placeholders = ",".join("?" for _ in doc_ids)
+        existing_rows = self._conn.execute(
+            f"SELECT doc_id, source_path FROM documents WHERE doc_id IN ({placeholders})",
+            doc_ids,
+        ).fetchall()
+        if not existing_rows:
+            return 0
+
+        existing_ids = [r[0] for r in existing_rows]
+        existing_paths = [r[1] for r in existing_rows if r[1]]
+        ph_e = ",".join("?" for _ in existing_ids)
+
+        with self._conn:
+            if self._use_fts5:
+                old_rowids = self._conn.execute(
+                    f"SELECT rowid FROM fts_nodes WHERE doc_id IN ({ph_e})",
+                    existing_ids,
+                ).fetchall()
+                if old_rowids:
+                    # Batch delete in chunks to avoid SQLite's variable limit (~999)
+                    _MAX_VARS = 500
+                    rowid_list = [r[0] for r in old_rowids]
+                    for i in range(0, len(rowid_list), _MAX_VARS):
+                        chunk = rowid_list[i:i + _MAX_VARS]
+                        ph2 = ",".join("?" for _ in chunk)
+                        self._conn.execute(
+                            f"DELETE FROM fts_nodes WHERE rowid IN ({ph2})",
+                            chunk,
+                        )
+            else:
+                self._conn.execute(
+                    f"DELETE FROM fts_nodes WHERE doc_id IN ({ph_e})", existing_ids
+                )
+            self._conn.execute(
+                f"DELETE FROM nodes WHERE doc_id IN ({ph_e})", existing_ids
+            )
+            self._conn.execute(
+                f"DELETE FROM documents WHERE doc_id IN ({ph_e})", existing_ids
+            )
+            self._conn.execute(
+                f"DELETE FROM pst_email_meta WHERE doc_id IN ({ph_e})", existing_ids
+            )
+            if existing_paths:
+                ph_p = ",".join("?" for _ in existing_paths)
+                self._conn.execute(
+                    f"DELETE FROM index_meta WHERE source_path IN ({ph_p})",
+                    existing_paths,
+                )
+
+        logger.info("Batch-deleted %d document(s)", len(existing_ids))
+        return len(existing_ids)
+
+    def get_doc_id_by_source_path(self, source_path: str) -> Optional[str]:
+        """Look up a doc_id from a source file path.
+
+        Useful when callers know the file path but not the internal doc_id.
+
+        Args:
+            source_path: absolute path of the source file.
+
+        Returns:
+            The ``doc_id`` string, or ``None`` if no document matches.
+        """
+        row = self._conn.execute(
+            "SELECT doc_id FROM documents WHERE source_path = ?", (source_path,)
+        ).fetchone()
+        return row[0] if row else None
+
+    def get_doc_ids_by_source_prefix(self, prefix: str) -> list[str]:
+        """Return doc_ids whose source_path starts with ``prefix``.
+
+        Used for multi-document sources (e.g. PST archives whose email docs
+        get derived paths ``<file>#<entry_id>``) — cascade delete / replace.
+        """
+        rows = self._conn.execute(
+            "SELECT doc_id FROM documents WHERE instr(source_path, ?) = 1",
+            (prefix,),
+        ).fetchall()
+        return [r[0] for r in rows]
+
+    def has_docs_with_source_prefix(self, prefix: str) -> bool:
+        """Whether any document's source_path starts with ``prefix``."""
+        row = self._conn.execute(
+            "SELECT 1 FROM documents WHERE instr(source_path, ?) = 1 LIMIT 1",
+            (prefix,),
+        ).fetchone()
+        return row is not None
+
+    def count_docs_with_source_prefix(self, prefix: str) -> int:
+        """Count documents whose source_path starts with ``prefix``."""
+        row = self._conn.execute(
+            "SELECT COUNT(*) FROM documents WHERE instr(source_path, ?) = 1",
+            (prefix,),
+        ).fetchone()
+        return row[0] if row else 0
+
+    def list_doc_names_by_source_prefix(self, prefix: str, limit: int) -> list[str]:
+        """List doc_names whose source_path starts with ``prefix`` (ordered, capped)."""
+        rows = self._conn.execute(
+            "SELECT doc_name FROM documents WHERE instr(source_path, ?) = 1 "
+            "ORDER BY doc_name LIMIT ?",
+            (prefix, limit),
+        ).fetchall()
+        return [r[0] for r in rows]
+
+    # -------------------------------------------------------------------
+    # PST 邮件元数据（ADR-0005：邮件列表分页 + 附件下载清单）
+    # -------------------------------------------------------------------
+
+    def upsert_email_meta(self, doc_id: str, pst_path: str, meta: dict) -> None:
+        """写入一封邮件的元数据（随派生文档同事务批次提交）。
+
+        Args:
+            doc_id: 派生文档 id（``<pst文档id>__<entry_id>``）
+            pst_path: 物理 PST 绝对路径（列表查询的分组键）
+            meta: pst_parser 产出的 email_meta dict
+        """
+        import json as _json
+
+        self._conn.execute(
+            """INSERT OR REPLACE INTO pst_email_meta
+               (doc_id, pst_path, entry_id, subject, sender, date, folder, attachments_json)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+            (
+                doc_id,
+                pst_path,
+                str(meta.get("entry_id", "")),
+                meta.get("subject", ""),
+                meta.get("sender", ""),
+                meta.get("date", ""),
+                meta.get("folder", ""),
+                _json.dumps(meta.get("attachments") or [], ensure_ascii=False),
+            ),
+        )
+
+    def list_email_meta(
+        self, pst_path: str, offset: int = 0, limit: int = 50
+    ) -> tuple[int, list[dict]]:
+        """分页列出一个 PST 的邮件元数据（日期倒序，无日期排尾）。
+
+        Returns:
+            (total, rows)：rows 含 entry_id/subject/sender/date/folder/doc_id。
+        """
+        total = self._conn.execute(
+            "SELECT COUNT(*) FROM pst_email_meta WHERE pst_path = ?", (pst_path,)
+        ).fetchone()[0]
+        rows = self._conn.execute(
+            """SELECT entry_id, subject, sender, date, folder, doc_id
+               FROM pst_email_meta WHERE pst_path = ?
+               ORDER BY CASE WHEN date = '' THEN 1 ELSE 0 END, date DESC, entry_id DESC
+               LIMIT ? OFFSET ?""",
+            (pst_path, limit, offset),
+        ).fetchall()
+        return total, [
+            {
+                "entry_id": r[0],
+                "subject": r[1],
+                "sender": r[2],
+                "date": r[3],
+                "folder": r[4],
+                "doc_id": r[5],
+            }
+            for r in rows
+        ]
+
+    def get_email_attachments(self, doc_id: str) -> list[dict] | None:
+        """按派生 doc_id 取附件清单（[{name,size,stored,filename}]）；无记录返回 None。"""
+        row = self._conn.execute(
+            "SELECT attachments_json FROM pst_email_meta WHERE doc_id = ?", (doc_id,)
+        ).fetchone()
+        return self._attachments_from_row(row)
+
+    def get_email_attachments_by_entry(
+        self, pst_path: str, entry_id: str
+    ) -> list[dict] | None:
+        """按 (PST 绝对路径, entry_id) 取附件清单（web 层不知道 doc_id 时用）。"""
+        row = self._conn.execute(
+            "SELECT attachments_json FROM pst_email_meta WHERE pst_path = ? AND entry_id = ?",
+            (pst_path, str(entry_id)),
+        ).fetchone()
+        return self._attachments_from_row(row)
+
+    @staticmethod
+    def _attachments_from_row(row) -> list[dict] | None:
+        import json as _json
+
+        if row is None:
+            return None
+        try:
+            return _json.loads(row[0] or "[]")
+        except _json.JSONDecodeError:
+            return []
+
+    # -------------------------------------------------------------------
+    # Index metadata (replaces _index_meta.json)
+    # -------------------------------------------------------------------
+
+    def get_index_meta(self, source_path: str) -> Optional[str]:
+        """Get the stored file hash for a source path.
+
+        Returns:
+            File hash string, or None if not tracked.
+        """
+        row = self._conn.execute(
+            "SELECT file_hash FROM index_meta WHERE source_path = ?",
+            (source_path,),
+        ).fetchone()
+        return row[0] if row else None
+
+    def set_index_meta(self, source_path: str, file_hash: str) -> None:
+        """Store/update the file hash for a source path."""
+        self._conn.execute(
+            "INSERT OR REPLACE INTO index_meta (source_path, file_hash) VALUES (?, ?)",
+            (source_path, file_hash),
+        )
+        self._conn.commit()
+
+    def set_index_meta_batch(self, meta: dict[str, str]) -> None:
+        """Batch store/update file hashes. Single transaction for performance."""
+        self._conn.executemany(
+            "INSERT OR REPLACE INTO index_meta (source_path, file_hash) VALUES (?, ?)",
+            list(meta.items()),
+        )
+        self._conn.commit()
+
+    def get_all_index_meta(self) -> dict[str, str]:
+        """Get all stored file hashes.
+
+        Returns:
+            Dict mapping source_path -> file_hash.
+        """
+        rows = self._conn.execute("SELECT source_path, file_hash FROM index_meta").fetchall()
+        return {r[0]: r[1] for r in rows}
+
+    # -------------------------------------------------------------------
+    # FTS5 query expression builder
+    # -------------------------------------------------------------------
+
+    @staticmethod
+    def build_fts_expression(
+        keywords: list[str],
+        operator: str = "OR",
+        column: Optional[str] = None,
+        near_distance: Optional[int] = None,
+        prefix: bool = False,
+    ) -> str:
+        """Build FTS5 match expression from keyword list.
+
+        Args:
+            keywords: list of search terms
+            operator: "AND" | "OR" | "NOT" (first keyword AND NOT others)
+            column: optional column filter (e.g. "title", "body")
+            near_distance: if set, uses NEAR(kw1 kw2, N) syntax
+            prefix: if True, append ``*`` to each term for prefix matching.
+                Prefix matching allows partial token matches: "conf*" matches
+                "config", "configuration", "confirm", etc. Useful for
+                autocomplete-style search or when query tokens may be substrings
+                of indexed terms (e.g. "fts*" matches both "fts" and "fts5").
+
+        Returns:
+            FTS5 match expression string
+
+        Examples::
+
+            build_fts_expression(["python", "async"], "AND")
+            -> "python AND async"
+
+            build_fts_expression(["fts"], prefix=True)
+            -> "fts*"   # matches fts, fts5, ftsearch, ...
+
+            build_fts_expression(["fts", "python"], prefix=True, operator="OR")
+            -> "fts* OR python*"
+
+            build_fts_expression(["machine", "learning"], column="title")
+            -> "title : (machine OR learning)"
+
+            build_fts_expression(["deep", "learning"], near_distance=5)
+            -> 'NEAR(deep learning, 5)'
+        """
+        if not keywords:
+            return ""
+
+        # Escape FTS5 special characters in keywords
+        safe_kws = []
+        for kw in keywords:
+            cleaned = _RE_FTS5_SPECIAL.sub("", kw.strip())
+            if cleaned:
+                # Tokenize for CJK
+                tokenized = _tokenize_for_fts(cleaned)
+                if tokenized.strip():
+                    term = tokenized.strip()
+                    if prefix:
+                        term = term + "*"
+                    safe_kws.append(term)
+
+        if not safe_kws:
+            return ""
+
+        if near_distance is not None and len(safe_kws) >= 2:
+            all_tokens = " ".join(safe_kws)
+            expr = f"NEAR({all_tokens}, {near_distance})"
+        elif operator == "NOT" and len(safe_kws) >= 2:
+            expr = f"{safe_kws[0]} NOT {' NOT '.join(safe_kws[1:])}"
+        else:
+            expr = f" {operator} ".join(safe_kws)
+
+        if column:
+            expr = f"{column} : ({expr})"
+
+        return expr
+
+    # -------------------------------------------------------------------
+    # Maintenance
+    # -------------------------------------------------------------------
+
+    def optimize(self) -> None:
+        """Run FTS5 merge optimization for better query performance."""
+        if not self._use_fts5:
+            logger.debug("FTS5 not available, skipping optimize")
+            return
+        try:
+            self._conn.execute("INSERT INTO fts_nodes(fts_nodes) VALUES('optimize')")
+            self._conn.commit()
+            logger.info("FTS5 index optimized")
+        except sqlite3.OperationalError as e:
+            logger.warning("FTS5 optimize failed: %s", e)
+
+    def rebuild(self) -> None:
+        """Rebuild FTS5 index from scratch."""
+        if not self._use_fts5:
+            logger.debug("FTS5 not available, skipping rebuild")
+            return
+        try:
+            self._conn.execute("INSERT INTO fts_nodes(fts_nodes) VALUES('rebuild')")
+            self._conn.commit()
+            logger.info("FTS5 index rebuilt")
+        except sqlite3.OperationalError as e:
+            logger.warning("FTS5 rebuild failed: %s", e)
+
+    def get_stats(self) -> dict:
+        """Get index statistics."""
+        doc_count = self._conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0]
+        node_count = self._conn.execute("SELECT COUNT(*) FROM nodes").fetchone()[0]
+        return {
+            "db_path": self._db_path,
+            "document_count": doc_count,
+            "node_count": node_count,
+        }
+
+    def clear(self) -> None:
+        """Clear all indexed data."""
+        self._conn.execute("DELETE FROM fts_nodes")
+        self._conn.execute("DELETE FROM nodes")
+        self._conn.execute("DELETE FROM documents")
+        self._conn.execute("DELETE FROM index_meta")
+        self._conn.commit()
+
+    def is_document_indexed(self, doc_id: str) -> bool:
+        """Check if a document is already indexed."""
+        row = self._conn.execute(
+            "SELECT 1 FROM documents WHERE doc_id = ?", (doc_id,)
+        ).fetchone()
+        return row is not None
+
+    def wal_checkpoint(self, mode: str = "TRUNCATE") -> None:
+        """Force a WAL checkpoint to fold the ``-wal`` sidecar back into the DB.
+
+        Useful at the end of a long incremental indexing run so the ``-wal``
+        file does not stay multi-GB after many small commits. ``TRUNCATE``
+        is the strongest variant (always safe — only no-ops when readers hold
+        the WAL open).
+        """
+        try:
+            self._conn.execute(f"PRAGMA wal_checkpoint({mode})")
+        except sqlite3.OperationalError as e:
+            logger.debug("WAL checkpoint(%s) failed: %s", mode, e)
+
+    def verify_index(self) -> dict:
+        """Cross-table consistency check.
+
+        Detects orphan rows that violate the four-table invariant:
+          - ``nodes`` rows whose doc_id has no entry in ``documents``.
+          - ``fts_nodes`` rows whose doc_id has no entry in ``documents``.
+          - ``index_meta`` rows whose source_path has no entry in ``documents``.
+          - ``documents`` rows with non-existent on-disk source_path.
+
+        Returns a report dict with ``healthy: bool`` and lists of problem ids.
+        Use :meth:`repair_index` to clean orphans.
+        """
+        report: dict = {
+            "healthy": True,
+            "orphan_node_doc_ids": [],
+            "orphan_fts_doc_ids": [],
+            "orphan_meta_paths": [],
+            "missing_source_paths": [],
+        }
+        cur = self._conn.execute(
+            "SELECT DISTINCT n.doc_id FROM nodes n "
+            "LEFT JOIN documents d ON n.doc_id = d.doc_id WHERE d.doc_id IS NULL"
+        ).fetchall()
+        report["orphan_node_doc_ids"] = [r[0] for r in cur]
+        cur = self._conn.execute(
+            "SELECT DISTINCT f.doc_id FROM fts_nodes f "
+            "LEFT JOIN documents d ON f.doc_id = d.doc_id WHERE d.doc_id IS NULL"
+        ).fetchall()
+        report["orphan_fts_doc_ids"] = [r[0] for r in cur]
+        cur = self._conn.execute(
+            "SELECT m.source_path FROM index_meta m "
+            "LEFT JOIN documents d ON m.source_path = d.source_path WHERE d.doc_id IS NULL"
+        ).fetchall()
+        report["orphan_meta_paths"] = [r[0] for r in cur]
+        cur = self._conn.execute(
+            "SELECT doc_id, source_path FROM documents WHERE source_path != ''"
+        ).fetchall()
+        report["missing_source_paths"] = [
+            (doc_id, sp) for (doc_id, sp) in cur if not os.path.isfile(sp)
+        ]
+        report["healthy"] = not any(
+            report[k] for k in
+            ("orphan_node_doc_ids", "orphan_fts_doc_ids",
+             "orphan_meta_paths", "missing_source_paths")
+        )
+        return report
+
+    def repair_index(self, drop_missing_files: bool = False) -> dict:
+        """Remove orphan rows surfaced by :meth:`verify_index`.
+
+        Args:
+            drop_missing_files: also delete documents whose source file no
+                longer exists on disk (default False — those may be intentional
+                in-memory loads or inaccessible mounts).
+
+        Returns:
+            Dict with counts of removed orphan rows.
+        """
+        report = self.verify_index()
+        removed = {"orphan_nodes": 0, "orphan_fts": 0, "orphan_meta": 0, "missing_files": 0}
+
+        with self._conn:
+            if report["orphan_node_doc_ids"]:
+                ph = ",".join("?" for _ in report["orphan_node_doc_ids"])
+                cur = self._conn.execute(
+                    f"DELETE FROM nodes WHERE doc_id IN ({ph})",
+                    report["orphan_node_doc_ids"],
+                )
+                removed["orphan_nodes"] = cur.rowcount
+            if report["orphan_fts_doc_ids"]:
+                ph = ",".join("?" for _ in report["orphan_fts_doc_ids"])
+                if self._use_fts5:
+                    rowids = self._conn.execute(
+                        f"SELECT rowid FROM fts_nodes WHERE doc_id IN ({ph})",
+                        report["orphan_fts_doc_ids"],
+                    ).fetchall()
+                    if rowids:
+                        ph2 = ",".join("?" for _ in rowids)
+                        cur = self._conn.execute(
+                            f"DELETE FROM fts_nodes WHERE rowid IN ({ph2})",
+                            [r[0] for r in rowids],
+                        )
+                        removed["orphan_fts"] = cur.rowcount
+                else:
+                    cur = self._conn.execute(
+                        f"DELETE FROM fts_nodes WHERE doc_id IN ({ph})",
+                        report["orphan_fts_doc_ids"],
+                    )
+                    removed["orphan_fts"] = cur.rowcount
+            if report["orphan_meta_paths"]:
+                ph = ",".join("?" for _ in report["orphan_meta_paths"])
+                cur = self._conn.execute(
+                    f"DELETE FROM index_meta WHERE source_path IN ({ph})",
+                    report["orphan_meta_paths"],
+                )
+                removed["orphan_meta"] = cur.rowcount
+
+        if drop_missing_files and report["missing_source_paths"]:
+            doc_ids = [doc_id for (doc_id, _) in report["missing_source_paths"]]
+            removed["missing_files"] = self.delete_documents(doc_ids)
+
+        return removed
+
+    def get_unindexed_doc_ids(self, doc_ids: list[str]) -> set[str]:
+        """Return the subset of doc_ids that are NOT yet indexed.
+
+        Uses a single SQL query instead of per-document checks.
+        """
+        if not doc_ids:
+            return set()
+        placeholders = ",".join("?" for _ in doc_ids)
+        rows = self._conn.execute(
+            f"SELECT doc_id FROM documents WHERE doc_id IN ({placeholders})",
+            doc_ids,
+        ).fetchall()
+        indexed = {r[0] for r in rows}
+        return set(doc_ids) - indexed
+
+
+def _extract_match_snippet(text: str, query: str, use_regex: bool, size: int = 300) -> str:
+    """Extract a snippet of *size* chars centered around the first match."""
+    if len(text) <= size:
+        return text
+    pos = -1
+    if use_regex:
+        m = re.search(query, text, re.IGNORECASE)
+        if m:
+            pos = m.start()
+    else:
+        pos = text.lower().find(query.lower())
+    if pos < 0:
+        pos = 0
+    half = size // 2
+    start = max(0, pos - half)
+    end = min(len(text), start + size)
+    start = max(0, end - size)
+    return text[start:end]
+
+
+# ---------------------------------------------------------------------------
+# Global FTS5 index singleton
+# ---------------------------------------------------------------------------
+
+_global_fts: Optional[FTS5Index] = None
+
+
+def get_fts_index(db_path: Optional[str] = None, weights: Optional[dict] = None) -> FTS5Index:
+    """Get or create the global FTS5 index.
+
+    Args:
+        db_path: database path. If None, uses in-memory database.
+                 Pass a file path for persistent indexing across sessions.
+        weights: column weight overrides for bm25() ranking.
+    """
+    global _global_fts
+    if _global_fts is not None:
+        # If db_path changed, re-create the singleton
+        requested = db_path or ":memory:"
+        if _global_fts.db_path != requested:
+            _global_fts.close()
+            _global_fts = None
+    if _global_fts is None:
+        _global_fts = FTS5Index(db_path=db_path, weights=weights)
+    return _global_fts
+
+
+def set_fts_index(index: FTS5Index) -> None:
+    """Set the global FTS5 index instance."""
+    global _global_fts
+    _global_fts = index
+
+
+def reset_fts_index() -> None:
+    """Close and reset the global FTS5 index."""
+    global _global_fts
+    if _global_fts is not None:
+        _global_fts.close()
+        _global_fts = None