treesearchlib 1.1.0__py3-none-any.whl

treesearch/indexer.py

@@ -0,0 +1,2038 @@
+# -*- coding: utf-8 -*-
+"""
+@author:XuMing(xuming624@qq.com)
+@description: Async-first document indexer. Builds tree structure from Markdown or plain text.
+
+Supports batch indexing via ``build_index()`` which accepts glob patterns and
+processes multiple files concurrently.
+"""
+import asyncio
+import hashlib
+import json
+import logging
+import os
+import re
+import time
+from contextlib import nullcontext
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from tqdm import tqdm
+
+from .tree import (
+    Document, assign_node_ids, flatten_tree, format_structure, remove_fields,
+)
+from .pathutil import resolve_paths, DEFAULT_IGNORE_DIRS, MAX_DIR_FILES, shadow_md_path
+
+logger = logging.getLogger(__name__)
+
+# PST 解析器输出格式版本盐（折进 PST 文件的指纹）：pst_parser 的输出格式
+# 变化（正文转写/元数据/附件落盘，ADR-0005）时 bump，让旧 PST 索引在下次
+# 增量索引自动重建——只影响 PST，不像 INDEX_SCHEMA_VERSION 那样连累全库。
+# 历史：
+#   ":pst2" — 2026-07-29 ADR-0005：HTML 正文转写 + 附件全量落盘 + 邮件元数据。
+#   ":pst3" — 2026-07-30：邮件头转储正文（Outlook 系统报告类）可读化重排
+#             （_reformat_header_dump：解码 encoded-word、折叠超长地址列表）。
+#   ":pst4" — 2026-07-30：头转储检测前移到 HTML 转写之前 + 逻辑块解析
+#             （body_html 装天书的路径不再漏检）。
+#   ":pst5" — 2026-07-30：头转储带完整 RFC822 源码时按 MIME 解析提取内嵌
+#             正文（base64 附件只列名）；头行正则兼容空值头（Subject:）。
+PST_PARSER_FINGERPRINT_SALT = ":pst5"
+
+
+# ============================================================================
+# Shared helpers
+# ============================================================================
+
+def _generate_shadow_md(binary_path: str) -> None:
+    """Convert a binary file to a hidden Markdown text copy using markitdown.
+
+    The shadow file (``._<name>.<ext>.md``) lives alongside the source and is
+    used by ripgrep fallback search when FTS5 has no results.
+
+    Skips generation if the shadow file is newer than the source (incremental).
+    """
+    md_path = shadow_md_path(binary_path)
+    # Incremental: skip if shadow is up-to-date
+    try:
+        if os.path.exists(md_path) and os.path.getmtime(md_path) >= os.path.getmtime(binary_path):
+            return
+    except OSError:
+        pass
+
+    try:
+        from markitdown import MarkItDown
+    except ImportError:
+        logger.debug("markitdown not installed, skipping shadow MD for %s", binary_path)
+        return
+
+    md = MarkItDown()
+    result = md.convert(binary_path)
+    text = result.text_content or ""
+    if not text.strip():
+        logger.debug("markitdown returned empty content for %s", binary_path)
+        return
+
+    try:
+        with open(md_path, "w", encoding="utf-8") as f:
+            f.write(text)
+        logger.debug("Generated shadow MD: %s", md_path)
+    except OSError as e:
+        logger.warning("Failed to write shadow MD %s: %s", md_path, e)
+
+
+def _children_indices(node_list: list[dict], parent_idx: int, parent_level: int) -> list[int]:
+    """Return indices of all descendants of node_list[parent_idx]."""
+    indices = []
+    for j in range(parent_idx + 1, len(node_list)):
+        if node_list[j]["level"] <= parent_level:
+            break
+        indices.append(j)
+    return indices
+
+
+# ============================================================================
+# Summary generation (shared by MD and Text)
+# ============================================================================
+
+def _summarize_node(node: dict, threshold: int = 600) -> str:
+    """Generate a summary for a single node. Short nodes use their own text.
+
+    Args:
+        threshold: character count threshold. Nodes shorter than this use full text as summary.
+            For long nodes: head 250 chars + tail 100 chars (captures intro and conclusion).
+    """
+    text = node.get("text", "")
+    if len(text) < threshold:
+        return text
+    head = text[:250].replace("\n", " ").strip()
+    tail = text[-100:].replace("\n", " ").strip()
+    return f"{head} ... {tail}"
+
+
+def generate_summaries(structure, threshold: int = 600):
+    """Generate summaries for all nodes in a tree."""
+    nodes = flatten_tree(structure)
+    summaries = [_summarize_node(n, threshold=threshold) for n in nodes]
+
+    for node, summary in zip(nodes, summaries):
+        if node.get("nodes"):
+            node["prefix_summary"] = summary
+        else:
+            node["summary"] = summary
+    return structure
+
+
+def generate_doc_description(structure) -> str:
+    """Generate a document description from its tree structure (no LLM).
+
+    Extracts top-level titles and first substantial text paragraph.
+    """
+    nodes = flatten_tree(structure)
+    titles = [n.get("title", "") for n in nodes if n.get("title")][:5]
+    title_str = " > ".join(titles)
+    text = ""
+    for n in nodes:
+        t = n.get("text", "")
+        if t and len(t) > 20:
+            text = t[:200]
+            break
+    return f"{title_str}. {text}" if text else title_str
+
+
+def _finalize_tree(
+    tree,
+    doc_name: str,
+    source_path: str = "",
+    source_type: str = "",
+    *,
+    if_add_node_id: bool = True,
+    if_add_node_summary: bool = True,
+    summary_chars_threshold: int = 600,
+    if_add_node_text: bool = False,
+    if_add_doc_description: bool = False,
+) -> dict:
+    """Common post-processing for all *_to_tree functions.
+
+    Steps: split_oversized_nodes -> assign_node_ids -> format_structure -> generate_summaries -> doc_description.
+    """
+    # Split oversized nodes before assigning IDs (so sub-nodes get proper IDs)
+    from .config import get_config
+    max_node_chars = get_config().max_node_chars
+    if max_node_chars:
+        tree = _split_oversized_nodes(tree, max_node_chars)
+
+    if if_add_node_id:
+        assign_node_ids(tree)
+
+    base_order = ["title", "node_id", "summary", "prefix_summary"]
+    text_fields = ["text"] if if_add_node_text or if_add_node_summary else []
+    tail_fields = ["line_start", "line_end", "nodes"]
+    order = base_order + text_fields + tail_fields
+
+    tree = format_structure(tree, order=order)
+
+    if if_add_node_summary:
+        logger.debug("Generating summaries...")
+        tree = generate_summaries(tree, threshold=summary_chars_threshold)
+        if not if_add_node_text:
+            order_no_text = [f for f in order if f != "text"]
+            tree = format_structure(tree, order=order_no_text)
+
+    result = {"doc_name": doc_name, "structure": tree}
+    if source_path:
+        result["source_path"] = source_path
+    if source_type:
+        result["source_type"] = source_type
+
+    if if_add_doc_description:
+        logger.debug("Generating document description...")
+        result["doc_description"] = generate_doc_description(tree)
+
+    return result
+
+
+# ============================================================================
+# Markdown indexer
+# ============================================================================
+
+def _extract_md_headings(content: str) -> tuple[list[dict], list[str]]:
+    """Extract heading markers from Markdown content."""
+    header_re = re.compile(r"^(#{1,6})\s+(.+)$")
+    code_fence = re.compile(r"^```")
+    markers = []
+    lines = content.split("\n")
+    in_code = False
+
+    for num, line in enumerate(lines, 1):
+        stripped = line.strip()
+        if code_fence.match(stripped):
+            in_code = not in_code
+            continue
+        if in_code or not stripped:
+            continue
+        m = header_re.match(stripped)
+        if m:
+            markers.append({
+                "title": m.group(2).strip(),
+                "line_num": num,
+                "level": len(m.group(1)),
+            })
+    return markers, lines
+
+
+def _cut_md_text(markers: list[dict], lines: list[str]) -> list[dict]:
+    """Cut text content between headings."""
+    nodes = []
+    for i, mk in enumerate(markers):
+        start = mk["line_num"] - 1
+        end = markers[i + 1]["line_num"] - 1 if i + 1 < len(markers) else len(lines)
+        nodes.append({
+            "title": mk["title"],
+            "line_num": mk["line_num"],
+            "line_start": mk["line_num"],
+            "line_end": end,
+            "level": mk["level"],
+            "text": "\n".join(lines[start:end]).strip(),
+        })
+    return nodes
+
+
+def _update_char_counts(node_list: list[dict]) -> list[dict]:
+    """Compute cumulative character counts (self + descendants) for thinning."""
+    for i in range(len(node_list) - 1, -1, -1):
+        text = node_list[i].get("text", "")
+        for ci in _children_indices(node_list, i, node_list[i]["level"]):
+            ct = node_list[ci].get("text", "")
+            if ct:
+                text += "\n" + ct
+        node_list[i]["text_char_count"] = len(text)
+    return node_list
+
+
+def _thin_tree(node_list: list[dict], min_chars: int) -> list[dict]:
+    """Merge small sub-trees into their parent nodes."""
+    to_remove = set()
+    for i in range(len(node_list) - 1, -1, -1):
+        if i in to_remove:
+            continue
+        if node_list[i].get("text_char_count", 0) < min_chars:
+            children = _children_indices(node_list, i, node_list[i]["level"])
+            merged_parts = []
+            for ci in sorted(children):
+                if ci not in to_remove:
+                    ct = node_list[ci].get("text", "")
+                    if ct.strip():
+                        merged_parts.append(ct)
+                    to_remove.add(ci)
+            if merged_parts:
+                base = node_list[i].get("text", "")
+                node_list[i]["text"] = base + "\n\n" + "\n\n".join(merged_parts) if base else "\n\n".join(merged_parts)
+                node_list[i]["text_char_count"] = len(node_list[i]["text"])
+
+    for idx in sorted(to_remove, reverse=True):
+        node_list.pop(idx)
+    return node_list
+
+
+def _build_tree(node_list: list[dict]) -> list[dict]:
+    """Build hierarchical tree from flat node list using a stack algorithm."""
+    if not node_list:
+        return []
+    stack = []
+    roots = []
+    counter = 1
+
+    for node in node_list:
+        level = node["level"]
+        tree_node = {
+            "title": node["title"],
+            "node_id": str(counter),
+            "text": node.get("text", ""),
+            "line_start": node.get("line_start", node.get("line_num")),
+            "line_end": node.get("line_end"),
+            "nodes": [],
+        }
+        counter += 1
+
+        while stack and stack[-1][1] >= level:
+            stack.pop()
+
+        if not stack:
+            roots.append(tree_node)
+        else:
+            stack[-1][0]["nodes"].append(tree_node)
+
+        stack.append((tree_node, level))
+    return roots
+
+
+# ============================================================================
+# Structure-aware node splitting for oversized nodes
+# ============================================================================
+
+def _split_text_by_paragraphs(text: str, max_chars: int) -> list[str]:
+    """Split text into chunks at paragraph boundaries (double newline).
+
+    Each chunk stays under max_chars. Falls back to single newline
+    boundaries, then hard character cut if paragraphs are still too large.
+    """
+    if len(text) <= max_chars:
+        return [text]
+
+    # Try splitting by double newline (paragraph boundary)
+    chunks = _split_at_boundary(text, max_chars, "\n\n")
+    if chunks:
+        return chunks
+
+    # Fallback: split by single newline (line boundary)
+    chunks = _split_at_boundary(text, max_chars, "\n")
+    if chunks:
+        return chunks
+
+    # Last resort: hard character cut (should rarely happen)
+    return [text[i:i + max_chars] for i in range(0, len(text), max_chars)]
+
+
+def _split_at_boundary(text: str, max_chars: int, separator: str) -> list[str]:
+    """Split text into chunks at the given separator, each under max_chars.
+
+    Returns None if any segment between separators exceeds max_chars
+    (caller should try a finer-grained separator).
+    """
+    segments = text.split(separator)
+    chunks = []
+    current = []
+    current_len = 0
+
+    for seg in segments:
+        seg_with_sep = (separator + seg) if current else seg
+        new_len = current_len + len(seg_with_sep)
+
+        if new_len > max_chars and current:
+            # Flush current chunk
+            chunks.append(separator.join(current))
+            current = [seg]
+            current_len = len(seg)
+            # If a single segment exceeds max_chars, this separator is too coarse
+            if current_len > max_chars:
+                return None
+        else:
+            current.append(seg)
+            current_len = new_len if current_len > 0 else len(seg)
+
+    if current:
+        chunks.append(separator.join(current))
+
+    return chunks if chunks else None
+
+
+def _split_oversized_nodes(tree: list[dict], max_chars: int) -> list[dict]:
+    """Recursively split oversized tree nodes into smaller sub-nodes.
+
+    For leaf nodes with text exceeding max_chars:
+    - Split text at paragraph boundaries (structure-aware)
+    - Create child nodes with sequential titles: "Part 1", "Part 2", etc.
+    - Parent node retains the original title with empty text
+
+    For non-leaf nodes: recurse into children first, then check the parent's
+    own text (the text before the first child heading).
+    """
+    if not max_chars:
+        return tree
+
+    result = []
+    for node in tree:
+        # Recurse into children first
+        children = node.get("nodes", [])
+        if children:
+            node["nodes"] = _split_oversized_nodes(children, max_chars)
+
+        text = node.get("text", "")
+        if len(text) <= max_chars:
+            result.append(node)
+            continue
+
+        # Node text exceeds max_chars: split into sub-nodes
+        chunks = _split_text_by_paragraphs(text, max_chars)
+        if len(chunks) <= 1:
+            result.append(node)
+            continue
+
+        title = node.get("title", "")
+        line_start = node.get("line_start")
+        line_end = node.get("line_end")
+        existing_children = node.get("nodes", [])
+
+        # Create sub-nodes from text chunks
+        sub_nodes = []
+        for i, chunk in enumerate(chunks):
+            sub_node = {
+                "title": f"{title} (part {i + 1})",
+                "text": chunk,
+                "line_start": line_start,
+                "line_end": line_end,
+                "nodes": [],
+            }
+            sub_nodes.append(sub_node)
+
+        # Attach existing children to the last sub-node (they belong to the tail of the text)
+        if existing_children:
+            sub_nodes[-1]["nodes"] = existing_children
+
+        # Parent node becomes a container with truncated text
+        node["text"] = ""
+        node["nodes"] = sub_nodes
+        result.append(node)
+
+    return result
+
+
+async def md_to_tree(
+    md_path: Optional[str] = None,
+    md_content: Optional[str] = None,
+    *,
+    if_thinning: bool = False,
+    min_thinning_chars: int = 15000,
+    if_add_node_summary: bool = True,
+    summary_chars_threshold: int = 600,
+    if_add_doc_description: bool = False,
+    if_add_node_text: bool = False,
+    if_add_node_id: bool = True,
+    **kwargs,
+) -> dict:
+    """
+    Build a tree index from a Markdown file or string.
+
+    Returns: {'doc_name': str, 'structure': list, 'doc_description'?: str}
+    """
+    if md_path and md_content:
+        raise ValueError("Specify only one of md_path or md_content")
+    if not md_path and not md_content:
+        raise ValueError("Must specify md_path or md_content")
+
+    if md_path:
+        with open(md_path, "r", encoding="utf-8", errors="replace") as f:
+            md_content = f.read()
+        doc_name = os.path.splitext(os.path.basename(md_path))[0]
+    else:
+        doc_name = "untitled"
+
+    logger.debug("Extracting headings from markdown...")
+    markers, lines = _extract_md_headings(md_content)
+    nodes = _cut_md_text(markers, lines)
+
+    if if_thinning and min_thinning_chars:
+        nodes = _update_char_counts(nodes)
+        logger.debug("Thinning tree (threshold=%d chars)...", min_thinning_chars)
+        nodes = _thin_tree(nodes, min_thinning_chars)
+
+    logger.debug("Building tree from %d nodes...", len(nodes))
+    tree = _build_tree(nodes)
+
+    # 无标题回退：为纯文本 Markdown 创建一个根节点
+    if not tree and md_content.strip():
+        total_lines = len(md_content.split("\n"))
+        tree = [{
+            "title": doc_name,
+            "node_id": "0",
+            "text": md_content.strip(),
+            "line_start": 1,
+            "line_end": total_lines,
+            "nodes": [],
+        }]
+
+    return _finalize_tree(
+        tree, doc_name,
+        source_path=os.path.abspath(md_path) if md_path else "",
+        if_add_node_id=if_add_node_id,
+        if_add_node_summary=if_add_node_summary,
+        summary_chars_threshold=summary_chars_threshold,
+        if_add_node_text=if_add_node_text,
+        if_add_doc_description=if_add_doc_description,
+    )
+
+
+# ============================================================================
+# Plain text indexer
+# ============================================================================
+
+# --- Heading detection patterns ---
+
+_RE_NUMERIC = re.compile(r"^(?P<prefix>(?:\d+\.)+\d*)\s*(?P<title>.+)$")
+_RE_PAREN_NUM = re.compile(r"^(?:\(?\d+\))\s+(?P<title>.+)$")
+_RE_ROMAN = re.compile(r"^(?P<prefix>[IVXLCDM]+)\.\s+(?P<title>.+)$")
+_RE_LETTER = re.compile(r"^(?P<prefix>[A-Z])[.)]\s+(?P<title>.+)$")
+_RE_CN_SECTION = re.compile(r"^(?:第[一二三四五六七八九十百千万零\d]+[章节篇部])\s*(?P<title>.*)$")
+_RE_CN_NUM = re.compile(r"^(?P<prefix>[一二三四五六七八九十百千万零]+)[、.．]\s*(?P<title>.+)$")
+_RE_CN_PAREN = re.compile(r"^[（(](?P<prefix>[一二三四五六七八九十百千万零\d]+)[)）]\s*(?P<title>.+)$")
+_RE_RST_UNDERLINE = re.compile(r"^[=\-~^+#]{3,}$")
+_RE_ALL_CAPS = re.compile(r"^[A-Z][A-Z\s\-:,&/]{2,}$")
+
+_ROMAN_VALID = {
+    "I", "II", "III", "IV", "V", "VI", "VII", "VIII", "IX", "X",
+    "XI", "XII", "XIII", "XIV", "XV", "XVI", "XVII", "XVIII", "XIX", "XX",
+}
+
+
+def _is_short(line: str, limit: int = 80) -> bool:
+    return 0 < len(line.strip()) < limit
+
+
+def _has_blank_neighbor(lines: list, idx: int) -> bool:
+    prev_blank = (idx == 0) or (not lines[idx - 1].strip())
+    next_blank = (idx >= len(lines) - 1) or (not lines[idx + 1].strip())
+    return prev_blank or next_blank
+
+
+def _detect_headings(lines: list[str]) -> list[dict]:
+    """Detect headings from raw text lines using pattern matching."""
+    headings = []
+    in_code = False
+
+    for idx, raw in enumerate(lines):
+        line = raw.strip()
+        if line.startswith("```"):
+            in_code = not in_code
+            continue
+        if in_code or not line:
+            continue
+        num = idx + 1
+
+        # Chinese chapter/section
+        m = _RE_CN_SECTION.match(line)
+        if m:
+            level = 1 if any(c in line for c in "章篇部") else 2
+            headings.append({"title": line, "line_num": num, "level": level})
+            continue
+
+        m = _RE_CN_NUM.match(line)
+        if m:
+            headings.append({"title": line, "line_num": num, "level": 1})
+            continue
+
+        m = _RE_CN_PAREN.match(line)
+        if m:
+            headings.append({"title": line, "line_num": num, "level": 2})
+            continue
+
+        # Numeric hierarchical (e.g. "1.2 Introduction", "3.1.1 Methods")
+        # Require short line and title starts with a letter to avoid matching
+        # math expressions like "0.1 + (-0.6) + 0.9 = 0.4..."
+        m = _RE_NUMERIC.match(line)
+        if m and _is_short(line) and re.match(r"[A-Za-z\u4e00-\u9fff]", m.group("title")):
+            level = len(m.group("prefix").rstrip(".").split("."))
+            headings.append({"title": line, "line_num": num, "level": level})
+            continue
+
+        # Parenthesized number
+        m = _RE_PAREN_NUM.match(line)
+        if m:
+            headings.append({"title": line, "line_num": num, "level": 2})
+            continue
+
+        # Roman numeral
+        m = _RE_ROMAN.match(line)
+        if m and m.group("prefix") in _ROMAN_VALID:
+            headings.append({"title": line, "line_num": num, "level": 1})
+            continue
+
+        # Letter heading (e.g. "A. Introduction", "B) Methods")
+        # Only match short lines to avoid false positives like "D. All these works..."
+        m = _RE_LETTER.match(line)
+        if m and _is_short(line, 60):
+            headings.append({"title": line, "line_num": num, "level": 2})
+            continue
+
+        # RST underline style
+        if idx > 0 and _RE_RST_UNDERLINE.match(line):
+            prev = lines[idx - 1].strip()
+            if prev and _is_short(prev):
+                if not headings or headings[-1]["line_num"] != idx:
+                    level = {"=": 1, "-": 2, "~": 3, "^": 4}.get(line[0], 2)
+                    headings.append({"title": prev, "line_num": idx, "level": level})
+                continue
+
+        # ALL CAPS
+        if _RE_ALL_CAPS.match(line) and _is_short(line) and _has_blank_neighbor(lines, idx):
+            headings.append({"title": line, "line_num": num, "level": 1})
+
+    return headings
+
+
+def _preprocess_text(text: str) -> str:
+    """Normalize line endings and collapse excessive blank lines."""
+    text = text.replace("\r\n", "\n").replace("\r", "\n").replace("\f", "\n")
+    return re.sub(r"\n{3,}", "\n\n", text)
+
+
+
+async def text_to_tree(
+    text_path: Optional[str] = None,
+    text_content: Optional[str] = None,
+    *,
+    if_thinning: bool = False,
+    min_thinning_chars: int = 15000,
+    if_add_node_summary: bool = True,
+    summary_chars_threshold: int = 600,
+    if_add_doc_description: bool = False,
+    if_add_node_text: bool = False,
+    if_add_node_id: bool = True,
+    **kwargs,
+) -> dict:
+    """
+    Build a tree index from plain text (pure rule-based, no LLM).
+
+    Args:
+        text_path: path to a .txt file
+        text_content: raw text string (alternative to text_path)
+    Returns:
+        {'doc_name': str, 'structure': list, 'doc_description'?: str}
+    """
+    if text_path and text_content:
+        raise ValueError("Specify only one of text_path or text_content")
+    if not text_path and not text_content:
+        raise ValueError("Must specify text_path or text_content")
+
+    if text_path:
+        with open(text_path, "r", encoding="utf-8", errors="replace") as f:
+            raw = f.read()
+        doc_name = os.path.splitext(os.path.basename(text_path))[0]
+    else:
+        raw = text_content
+        doc_name = "untitled"
+
+    text = _preprocess_text(raw)
+    lines = text.split("\n")
+    logger.debug("Text loaded: %d lines", len(lines))
+
+    # Step 1: heading detection (pure rule-based)
+    headings = _detect_headings(lines)
+    markers = [{"title": h["title"], "line_num": h["line_num"], "level": h["level"]} for h in headings]
+    logger.debug("Rule-based detection: %d headings", len(markers))
+
+    # Fallback: single root node if no headings detected
+    if not markers:
+        markers = [{"title": doc_name, "line_num": 1, "level": 1}]
+
+    # Step 2: extract text
+    nodes = _cut_md_text(markers, lines)
+
+    # Step 3: thinning
+    if if_thinning and min_thinning_chars:
+        nodes = _update_char_counts(nodes)
+        logger.debug("Thinning tree (threshold=%d chars)...", min_thinning_chars)
+        nodes = _thin_tree(nodes, min_thinning_chars)
+
+    # Step 4: build tree
+    logger.debug("Building tree from %d nodes...", len(nodes))
+    tree = _build_tree(nodes)
+
+    return _finalize_tree(
+        tree, doc_name,
+        source_path=os.path.abspath(text_path) if text_path else "",
+        if_add_node_id=if_add_node_id,
+        if_add_node_summary=if_add_node_summary,
+        summary_chars_threshold=summary_chars_threshold,
+        if_add_node_text=if_add_node_text,
+        if_add_doc_description=if_add_doc_description,
+    )
+
+
+# ============================================================================
+# Code file indexer
+# ============================================================================
+
+def _detect_code_headings(lines: list[str], ext: str, source: str = "") -> list[dict]:
+    """Detect classes and methods from code lines.
+
+    For ``.py`` files, tries AST-based parsing first (richer signatures),
+    falling back to regex if AST fails (e.g. syntax errors).
+    """
+    # Python: use AST parser for accurate structure extraction
+    if ext == ".py" and source:
+        from .parsers.ast_parser import parse_python_structure
+        headings = parse_python_structure(source)
+        if headings:
+            return headings
+        # AST failed, fall through to regex
+
+    headings = []
+    patterns = []
+    if ext == ".py":
+        patterns = [
+            (re.compile(r"^(class\s+\w+.*)"), 1),
+            (re.compile(r"^(\s*def\s+\w+.*)"), 2)
+        ]
+    elif ext in (".java", ".ts", ".js", ".cpp", ".cc", ".cs", ".php"):
+        patterns = [
+            (re.compile(r"^(\s*(?:public|private|protected|static|abstract|final\s+)*class\s+\w+.*)"), 1),
+            (re.compile(r"^(\s*(?:public|private|protected|static|abstract|final\s+)*interface\s+\w+.*)"), 1),
+            (re.compile(r"^(\s*(?:public|private|protected|static|abstract|final\s+)*(?:[\w<>\[\]]+\s+)+\w+\s*\(.*)"), 2),
+            (re.compile(r"^(\s*function\s+\w+.*)"), 2)
+        ]
+    elif ext == ".go":
+        patterns = [
+            (re.compile(r"^(\s*type\s+\w+\s+struct.*)"), 1),
+            (re.compile(r"^(\s*type\s+\w+\s+interface.*)"), 1),
+            (re.compile(r"^(\s*func\s+(?:\([^)]+\)\s+)?\w+.*)"), 2)
+        ]
+    elif ext == ".html":
+        patterns = [
+            (re.compile(r"^\s*<h1.*>(.*)</h1>"), 1),
+            (re.compile(r"^\s*<h2.*>(.*)</h2>"), 2),
+            (re.compile(r"^\s*<h3.*>(.*)</h3>"), 3),
+            (re.compile(r"^\s*<div.*id=\"(.*)\".*>"), 2),
+            (re.compile(r"^\s*<section.*id=\"(.*)\".*>"), 2)
+        ]
+    elif ext == ".xml":
+        patterns = [
+            (re.compile(r"^\s*<(\w+).*>\s*$"), 1),
+        ]
+
+    if not patterns:
+        return []
+
+    for idx, raw in enumerate(lines):
+        line = raw.rstrip()
+        if not line:
+            continue
+        num = idx + 1
+
+        for pat, level in patterns:
+            m = pat.match(line)
+            if m:
+                title = m.group(1).strip().rstrip(":{").strip()[:100]
+                headings.append({"title": title, "line_num": num, "level": level})
+                break
+
+    return headings
+
+
+async def code_to_tree(
+    code_path: str,
+    *,
+    if_thinning: bool = False,
+    min_thinning_chars: int = 15000,
+    if_add_node_summary: bool = True,
+    summary_chars_threshold: int = 600,
+    if_add_doc_description: bool = False,
+    if_add_node_text: bool = False,
+    if_add_node_id: bool = True,
+    **kwargs,
+) -> dict:
+    """
+    Build a tree index from a code file.
+
+    Returns:
+        {'doc_name': str, 'structure': list, 'doc_description'?: str}
+    """
+    with open(code_path, "r", encoding="utf-8", errors="replace") as f:
+        raw = f.read()
+    doc_name = os.path.splitext(os.path.basename(code_path))[0]
+    ext = os.path.splitext(code_path)[1].lower()
+
+    text = raw.replace("\r\n", "\n").replace("\r", "\n")
+    lines = text.split("\n")
+    logger.debug("Code loaded: %d lines", len(lines))
+
+    headings = _detect_code_headings(lines, ext, source=text)
+    markers = [{"title": h["title"], "line_num": h["line_num"], "level": h["level"]} for h in headings]
+    logger.debug("Code structure detection: %d methods/classes", len(markers))
+
+    if not markers:
+        markers = [{"title": doc_name, "line_num": 1, "level": 1}]
+
+    nodes = _cut_md_text(markers, lines)
+
+    if if_thinning and min_thinning_chars:
+        nodes = _update_char_counts(nodes)
+        logger.debug("Thinning tree (threshold=%d chars)...", min_thinning_chars)
+        nodes = _thin_tree(nodes, min_thinning_chars)
+
+    logger.debug("Building tree from %d nodes...", len(nodes))
+    tree = _build_tree(nodes)
+
+    return _finalize_tree(
+        tree, doc_name,
+        source_path=os.path.abspath(code_path),
+        if_add_node_id=if_add_node_id,
+        if_add_node_summary=if_add_node_summary,
+        summary_chars_threshold=summary_chars_threshold,
+        if_add_node_text=if_add_node_text,
+        if_add_doc_description=if_add_doc_description,
+    )
+
+
+# ============================================================================
+# JSON file indexer
+# ============================================================================
+
+def _json_to_nodes(data, prefix: str = "", level: int = 1) -> list[dict]:
+    """Recursively convert JSON data into flat node list."""
+    nodes = []
+    if isinstance(data, dict):
+        for key, value in data.items():
+            path = f"{prefix}.{key}" if prefix else key
+            if isinstance(value, (dict, list)):
+                nodes.append({"title": path, "level": level, "text": ""})
+                nodes.extend(_json_to_nodes(value, prefix=path, level=level + 1))
+            else:
+                nodes.append({"title": path, "level": level, "text": f"{key}: {value}"})
+    elif isinstance(data, list):
+        for i, item in enumerate(data):
+            path = f"{prefix}[{i}]"
+            if isinstance(item, (dict, list)):
+                nodes.append({"title": path, "level": level, "text": ""})
+                nodes.extend(_json_to_nodes(item, prefix=path, level=level + 1))
+            else:
+                nodes.append({"title": path, "level": level, "text": str(item)})
+    return nodes
+
+
+def _load_json_lenient(raw: str):
+    """Load JSON tolerantly: handle control characters, comments, and trailing commas."""
+    # First try with strict=False to tolerate control characters
+    try:
+        return json.loads(raw, strict=False)
+    except json.JSONDecodeError:
+        pass
+
+    # Strip trailing commas before } or ]
+    stripped = re.sub(r',\s*([}\]])', r'\1', raw)
+    # Strip // comments, but preserve // inside quoted strings (e.g. URLs)
+    stripped = re.sub(r'("(?:[^"\\]|\\.)*")|//[^\n]*', r'\1', stripped)
+    return json.loads(stripped, strict=False)
+
+
+async def json_to_tree(
+    json_path: str,
+    *,
+    if_add_node_summary: bool = True,
+    summary_chars_threshold: int = 600,
+    if_add_doc_description: bool = False,
+    if_add_node_text: bool = False,
+    if_add_node_id: bool = True,
+    **kwargs,
+) -> dict:
+    """Build a tree index from a JSON file."""
+    with open(json_path, "r", encoding="utf-8", errors="replace") as f:
+        raw = f.read()
+    doc_name = os.path.splitext(os.path.basename(json_path))[0]
+
+    try:
+        data = _load_json_lenient(raw)
+    except (json.JSONDecodeError, ValueError):
+        # JSON 解析失败，降级为纯文本
+        logger.debug("JSON parse failed for %s, falling back to text", json_path)
+        result = await text_to_tree(
+            text_content=raw,
+            if_add_node_summary=if_add_node_summary,
+            summary_chars_threshold=summary_chars_threshold,
+            if_add_doc_description=if_add_doc_description,
+            if_add_node_text=if_add_node_text,
+            if_add_node_id=if_add_node_id,
+            **kwargs,
+        )
+        result["doc_name"] = doc_name
+        result["source_path"] = os.path.abspath(json_path)
+        return result
+
+    flat_nodes = _json_to_nodes(data)
+    if not flat_nodes:
+        flat_nodes = [{"title": doc_name, "level": 1, "text": json.dumps(data, ensure_ascii=False)[:500]}]
+
+    # Assign line_num for _build_tree compatibility
+    for i, node in enumerate(flat_nodes):
+        node["line_num"] = i + 1
+        node["line_start"] = i + 1
+        node["line_end"] = i + 1
+
+    tree = _build_tree(flat_nodes)
+
+    return _finalize_tree(
+        tree, doc_name,
+        source_path=os.path.abspath(json_path),
+        if_add_node_id=if_add_node_id,
+        if_add_node_summary=if_add_node_summary,
+        summary_chars_threshold=summary_chars_threshold,
+        if_add_node_text=if_add_node_text,
+        if_add_doc_description=if_add_doc_description,
+    )
+
+
+# ============================================================================
+# JSONL file indexer
+# ============================================================================
+
+def _jsonl_to_nodes(records: list[dict], key_field: str = None) -> list[dict]:
+    """Convert a list of JSONL records into flat node list.
+
+    Each record becomes a level-1 node. If key_field is specified and exists
+    in the record, it is used as the title; otherwise uses the record index.
+    Nested structures within each record are expanded as child nodes.
+    """
+    nodes = []
+    for i, record in enumerate(records):
+        # Determine title for this record
+        if key_field and isinstance(record, dict) and key_field in record:
+            title = str(record[key_field])
+        elif isinstance(record, dict):
+            # Auto-detect: use first string-valued field as title
+            title = None
+            for k, v in record.items():
+                if isinstance(v, str) and len(v) < 200:
+                    title = f"{k}: {v}"
+                    break
+            if title is None:
+                title = f"record[{i}]"
+        else:
+            title = f"record[{i}]"
+
+        # Build text from record content
+        if isinstance(record, dict):
+            text_parts = []
+            child_nodes = []
+            for k, v in record.items():
+                if isinstance(v, (dict, list)):
+                    child_nodes.extend(_json_to_nodes(v, prefix=f"record[{i}].{k}", level=2))
+                else:
+                    text_parts.append(f"{k}: {v}")
+            text = "\n".join(text_parts)
+            nodes.append({"title": title, "level": 1, "text": text})
+            nodes.extend(child_nodes)
+        else:
+            nodes.append({"title": title, "level": 1, "text": str(record)})
+
+    return nodes
+
+
+async def jsonl_to_tree(
+    jsonl_path: str,
+    *,
+    key_field: str = None,
+    if_add_node_summary: bool = True,
+    summary_chars_threshold: int = 600,
+    if_add_doc_description: bool = False,
+    if_add_node_text: bool = False,
+    if_add_node_id: bool = True,
+    **kwargs,
+) -> dict:
+    """Build a tree index from a JSONL file (one JSON object per line).
+
+    Args:
+        jsonl_path: path to the .jsonl file
+        key_field: optional field name to use as record title
+    """
+    records = []
+    with open(jsonl_path, "r", encoding="utf-8", errors="replace") as f:
+        for line_num, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError as e:
+                logger.warning("Skipping invalid JSON at line %d in %s: %s", line_num, jsonl_path, e)
+
+    doc_name = os.path.splitext(os.path.basename(jsonl_path))[0]
+    logger.debug("JSONL loaded: %d records from %s", len(records), jsonl_path)
+
+    flat_nodes = _jsonl_to_nodes(records, key_field=key_field)
+    if not flat_nodes:
+        flat_nodes = [{"title": doc_name, "level": 1, "text": ""}]
+
+    for i, node in enumerate(flat_nodes):
+        node["line_num"] = i + 1
+        node["line_start"] = i + 1
+        node["line_end"] = i + 1
+
+    tree = _build_tree(flat_nodes)
+
+    return _finalize_tree(
+        tree, doc_name,
+        source_path=os.path.abspath(jsonl_path),
+        if_add_node_id=if_add_node_id,
+        if_add_node_summary=if_add_node_summary,
+        summary_chars_threshold=summary_chars_threshold,
+        if_add_node_text=if_add_node_text,
+        if_add_doc_description=if_add_doc_description,
+    )
+
+
+# ============================================================================
+# CSV file indexer
+# ============================================================================
+
+async def csv_to_tree(
+    csv_path: str,
+    *,
+    if_add_node_summary: bool = True,
+    summary_chars_threshold: int = 600,
+    if_add_doc_description: bool = False,
+    if_add_node_text: bool = False,
+    if_add_node_id: bool = True,
+    **kwargs,
+) -> dict:
+    """Build a tree index from a CSV file. Each row becomes a leaf node under a header node."""
+    import csv as csvmod
+
+    with open(csv_path, "r", encoding="utf-8", errors="replace") as f:
+        reader = csvmod.reader(f)
+        rows = list(reader)
+
+    doc_name = os.path.splitext(os.path.basename(csv_path))[0]
+    if not rows:
+        return {"doc_name": doc_name, "structure": [{"title": doc_name, "node_id": "0001", "nodes": []}]}
+
+    headers = rows[0]
+    flat_nodes = [{"title": doc_name, "level": 1, "text": f"Columns: {', '.join(headers)}", "line_num": 1, "line_start": 1, "line_end": 1}]
+
+    for i, row in enumerate(rows[1:], start=2):
+        row_text = "; ".join(f"{h}: {v}" for h, v in zip(headers, row) if v.strip())
+        title = row_text[:80] if row_text else f"Row {i}"
+        flat_nodes.append({"title": title, "level": 2, "text": row_text, "line_num": i, "line_start": i, "line_end": i})
+
+    tree = _build_tree(flat_nodes)
+
+    return _finalize_tree(
+        tree, doc_name,
+        source_path=os.path.abspath(csv_path),
+        if_add_node_id=if_add_node_id,
+        if_add_node_summary=if_add_node_summary,
+        summary_chars_threshold=summary_chars_threshold,
+        if_add_node_text=if_add_node_text,
+        if_add_doc_description=if_add_doc_description,
+    )
+
+
+# ============================================================================
+# Index statistics
+# ============================================================================
+
+@dataclass
+class IndexStats:
+    """Statistics collected during an indexing run.
+
+    Attributes:
+        total_files: Total files discovered (including skipped).
+        indexed_files: Files actually (re-)indexed in this run.
+        skipped_files: Files skipped because they were unchanged.
+        failed_files: Files that failed to parse.
+        total_nodes: Total tree nodes generated across all indexed files.
+        total_time_s: Total wall-clock time for the indexing run.
+        per_type: Breakdown by source_type with counts, node totals, and timings.
+        db_path: Path to the SQLite database file.
+        db_size_bytes: Size of the database file on disk (0 for in-memory).
+        failed_paths: List of file paths that failed to index.
+        node_diff: Aggregate node-level diff across reindexed docs:
+            ``{"added", "changed", "removed", "kept"}``.
+            Useful for verifying incremental behaviour.
+        pruned_paths: Source paths whose documents were removed from the
+            index because the file no longer exists in the indexed scope.
+    """
+    total_files: int = 0
+    indexed_files: int = 0
+    skipped_files: int = 0
+    failed_files: int = 0
+    excluded_files: int = 0
+    total_nodes: int = 0
+    total_time_s: float = 0.0
+    per_type: dict = field(default_factory=dict)
+    db_path: str = ""
+    db_size_bytes: int = 0
+    failed_paths: list = field(default_factory=list)
+    node_diff: dict = field(
+        default_factory=lambda: {"added": 0, "changed": 0, "removed": 0, "kept": 0}
+    )
+    pruned_paths: list = field(default_factory=list)
+
+    def summary(self) -> str:
+        """Return a human-readable summary string."""
+        lines = []
+        lines.append(f"Index Statistics")
+        lines.append(f"  Total files discovered: {self.total_files}")
+        lines.append(f"  Indexed (new/changed):  {self.indexed_files}")
+        lines.append(f"  Skipped (unchanged):    {self.skipped_files}")
+        if self.failed_files:
+            lines.append(f"  Failed:                 {self.failed_files}")
+        if self.excluded_files:
+            lines.append(f"  Exceeded failures:      {self.excluded_files}")
+        if self.pruned_paths:
+            lines.append(f"  Pruned (orphans):       {len(self.pruned_paths)}")
+        lines.append(f"  Total nodes generated:  {self.total_nodes}")
+        nd = self.node_diff
+        if any(nd.values()):
+            lines.append(
+                f"  Node-level diff:        +{nd['added']}  ~{nd['changed']}  "
+                f"-{nd['removed']}  (kept {nd['kept']})"
+            )
+        lines.append(f"  Total time:             {self.total_time_s:.3f}s")
+        if self.db_path:
+            size_str = _format_size(self.db_size_bytes)
+            lines.append(f"  Database:               {self.db_path} ({size_str})")
+
+        if self.per_type:
+            lines.append(f"")
+            lines.append(f"  Per file type:")
+            # Sort by file count descending
+            for stype, info in sorted(self.per_type.items(), key=lambda x: -x[1]["count"]):
+                cnt = info["count"]
+                nodes = info["nodes"]
+                t = info["time_s"]
+                lines.append(f"    {stype:12s}  {cnt:4d} file(s)  {nodes:5d} nodes  {t:.3f}s")
+
+        if self.failed_paths:
+            lines.append(f"")
+            lines.append(f"  Failed files:")
+            for fp in self.failed_paths[:10]:
+                lines.append(f"    - {fp}")
+            if len(self.failed_paths) > 10:
+                lines.append(f"    ... and {len(self.failed_paths) - 10} more")
+
+        return "\n".join(lines)
+
+
+def _format_size(size_bytes: int) -> str:
+    """Format bytes into human-readable string."""
+    if size_bytes < 1024:
+        return f"{size_bytes} B"
+    elif size_bytes < 1024 * 1024:
+        return f"{size_bytes / 1024:.1f} KB"
+    else:
+        return f"{size_bytes / (1024 * 1024):.1f} MB"
+
+
+# ============================================================================
+# Batch indexing API
+# ============================================================================
+
+class _NullLock:
+    """No-op lock context for in-memory or non-POSIX paths."""
+    def __enter__(self):
+        return self
+    def __exit__(self, *a):
+        return False
+    def release(self) -> None:
+        return None
+
+
+def _acquire_index_lock(db_path: str):
+    """Acquire an exclusive advisory lock on ``{db_path}.lock``.
+
+    Returns a handle whose ``release()`` method closes the file (which
+    releases the lock). Returns a no-op handle for in-memory or non-existent
+    paths, and on platforms without ``fcntl`` (e.g. Windows).
+    """
+    if not db_path or db_path == ":memory:":
+        return _NullLock()
+
+    # Unix: 使用 fcntl.flock
+    try:
+        import fcntl
+    except ImportError:
+        pass
+    else:
+        lock_path = db_path + ".lock"
+        os.makedirs(os.path.dirname(os.path.abspath(lock_path)), exist_ok=True)
+        f = open(lock_path, "w")
+        try:
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+        except OSError as e:
+            f.close()
+            logger.warning("Failed to acquire index lock %s: %s", lock_path, e)
+            return _NullLock()
+
+        class _Handle:
+            def __init__(self, fh):
+                self._fh = fh
+            def __enter__(self):
+                return self
+            def __exit__(self, *a):
+                self.release()
+                return False
+            def release(self) -> None:
+                if self._fh is not None:
+                    try:
+                        fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN)
+                    finally:
+                        self._fh.close()
+                        self._fh = None
+
+        return _Handle(f)
+
+    # Windows: 使用 msvcrt.locking
+    try:
+        import msvcrt
+    except ImportError:
+        logger.warning("No file locking available on this platform (no fcntl, no msvcrt)")
+        return _NullLock()
+
+    lock_path = db_path + ".lock"
+    os.makedirs(os.path.dirname(os.path.abspath(lock_path)), exist_ok=True)
+    f = open(lock_path, "w+")
+
+    class _WindowsHandle:
+        def __init__(self, fh):
+            self._fh = fh
+        def __enter__(self):
+            return self
+        def __exit__(self, *a):
+            self.release()
+            return False
+        def release(self) -> None:
+            if self._fh is not None:
+                try:
+                    msvcrt.locking(self._fh.fileno(), msvcrt.LK_UNLCK, 0)
+                except (OSError, IOError):
+                    pass  # Lock auto-released on close on Windows
+                finally:
+                    self._fh.close()
+                    self._fh = None
+
+    import time as _time
+    # 重试：瞬态锁冲突（同进程内并发的 build_index）不该让本次索引直接放弃并返回 []
+    # （会把 documents 清空，导致 files API 的 indexed 标志全部消失）。
+    last_err = None
+    for _attempt in range(10):
+        try:
+            msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1)
+            return _WindowsHandle(f)
+        except (OSError, IOError) as e:
+            last_err = e
+            _time.sleep(0.2)
+    f.close()
+    logger.warning("Failed to acquire Windows index lock %s after retries: %s", lock_path, last_err)
+    return _NullLock()
+
+
+def file_hash(fp: str, mode: Optional[str] = None) -> str:
+    """Compute a fingerprint for incremental indexing.
+
+    Format: ``"v{INDEX_SCHEMA_VERSION}:{mode}:{payload}"`` so that:
+      - bumping ``INDEX_SCHEMA_VERSION`` invalidates every stored hash, forcing a
+        clean rebuild after a parser/tokenizer/schema change.
+      - switching ``fingerprint_mode`` between ``stat`` and ``content`` likewise
+        invalidates so users can opt-in safely.
+
+    Modes:
+      - ``stat`` (default): ``(mtime_ns:size)`` — fast, catches normal edits.
+      - ``content``: full md5 for files <``content_fingerprint_size_threshold``;
+        large files are sampled at head/middle/tail with
+        ``content_fingerprint_sample_bytes`` per region. Robust against
+        ``touch``/CI-replay scenarios at the cost of one read per file.
+
+    Returns empty string if the file does not exist.
+    """
+    from .config import INDEX_SCHEMA_VERSION, get_config
+    cfg = get_config()
+    if mode is None:
+        mode = cfg.fingerprint_mode
+
+    try:
+        st = os.stat(fp)
+    except (FileNotFoundError, OSError):
+        return ""
+
+    if mode == "content":
+        threshold = cfg.content_fingerprint_size_threshold
+        sample = cfg.content_fingerprint_sample_bytes
+        h = hashlib.md5()
+        try:
+            with open(fp, "rb") as f:
+                if st.st_size <= threshold:
+                    h.update(f.read())
+                else:
+                    h.update(f.read(sample))
+                    if st.st_size > 2 * sample:
+                        f.seek(st.st_size // 2)
+                        h.update(f.read(sample))
+                    f.seek(max(0, st.st_size - sample))
+                    h.update(f.read(sample))
+        except OSError:
+            return ""
+        payload = f"{st.st_size}:{h.hexdigest()}"
+    else:
+        payload = f"{st.st_mtime_ns}:{st.st_size}"
+
+    return f"v{INDEX_SCHEMA_VERSION}:{mode}:{payload}"
+
+
+def file_hash_with_salts(fp: str, mode: Optional[str] = None) -> str:
+    """``file_hash`` + 解析器格式盐（插在版本号后，保持末尾 size 可解析）。
+
+    PST 解析器输出格式变化（ADR-0005）时 bump PST_PARSER_FINGERPRINT_SALT，
+    旧 PST 索引自动重建，不像 INDEX_SCHEMA_VERSION 那样连累全库。
+    增量比较与移动检测都必须用本函数（口径一致才能匹配）。
+
+    图像文件（jpg/jpeg/png/webp）改用解码像素指纹：写回 EXIF/XMP 元数据不改像素，
+    故不触发「写回 → hash 变 → 增量重解析」死循环（ADR-0009 / 工单 03）。解码失败
+    回退到 stat/content。注：口径切换使旧图像 hash 失效 → 首迁移重索引（已知代价）。
+    """
+    ext = os.path.splitext(fp)[1].lower()
+    if ext in (".jpg", ".jpeg", ".png", ".webp"):  # 与 image_metadata.INTERPRETED_IMAGE_EXTS 一致
+        try:
+            from .config import INDEX_SCHEMA_VERSION
+            from .parsers.image_metadata import content_fingerprint
+            return f"v{INDEX_SCHEMA_VERSION}:image:{content_fingerprint(fp)}"
+        except Exception:
+            pass  # 解码失败回退到 stat/content
+    h = file_hash(fp, mode)
+    if h and fp.lower().endswith(".pst"):
+        ver, sep, rest = h.partition(":")
+        return f"{ver}{sep}{PST_PARSER_FINGERPRINT_SALT.strip(':')}:{rest}"
+    return h
+
+
+def _extract_size_from_fingerprint(fp_hash: str) -> Optional[str]:
+    """Extract the file size portion from a fingerprint string.
+
+    Supports both stat (``v{VER}:stat:{mtime}:{size}``) and content
+    (``v{VER}:content:{size}:{md5}``) formats.
+    """
+    parts = fp_hash.split(":")
+    if len(parts) < 3:
+        return None
+    candidate = parts[-1]
+    return candidate if candidate.isdigit() else None
+
+
+def _detect_and_apply_moves(
+    expanded: list[str],
+    all_meta: dict[str, str],
+    fts,
+    fp_to_doc_id: dict[str, str],
+    file_hash_fn,
+) -> list[tuple[str, str]]:
+    """Detect file moves/renames and apply them in-place via ``fts.rename_document``.
+
+    Two-pass strategy:
+
+    1. **Fingerprint match** (exact): compute the file fingerprint and look it up
+       in a reverse index built from stored metadata.  This works perfectly when
+       the fingerprint is content-based (MD5), and also catches the rare case
+       where ``stat`` mtime is preserved after a move.
+
+    2. **Size heuristic** (fallback): for stat-mode fingerprints where mtime
+       changed on move, match by ``(basename, file_size)``.  This is reliable
+       because: same basename + same byte size + old path gone = move.
+
+    Mutates ``all_meta`` in-place by popping old entries and inserting new ones.
+    Returns a list of ``(old_path, new_path)`` tuples for logging.
+    """
+    from .config import INDEX_SCHEMA_VERSION
+
+    _current_prefix = f"v{INDEX_SCHEMA_VERSION}:"
+
+    # --- Build reverse index: fingerprint → stored_path ---
+    hash_to_old_path: dict[str, str] = {}
+    for sp, fh in all_meta.items():
+        if fh.startswith(_current_prefix):
+            hash_to_old_path.setdefault(fh, sp)
+
+    moved: list[tuple[str, str]] = []
+    expanded_abs = {os.path.abspath(p) for p in expanded}
+    matched_new = set()  # abs_fp of new files already claimed
+
+    # --- Pass 1: fingerprint-based matching ---
+    if hash_to_old_path:
+        for fp in expanded:
+            abs_fp = os.path.abspath(fp)
+            if abs_fp in all_meta:
+                continue
+            fh = file_hash_fn(abs_fp)
+            if not fh:
+                continue
+            old_path = hash_to_old_path.get(fh)
+            if old_path and old_path != abs_fp and not os.path.isfile(old_path):
+                moved_doc_id = fts.get_doc_id_by_source_path(old_path)
+                if moved_doc_id:
+                    new_doc_id = fp_to_doc_id.get(fp, os.path.splitext(os.path.basename(abs_fp))[0])
+                    if fts.rename_document(moved_doc_id, new_doc_id, new_doc_id, abs_fp):
+                        fts.set_index_meta(abs_fp, fh)
+                        all_meta.pop(old_path, None)
+                        all_meta[abs_fp] = fh
+                        hash_to_old_path[fh] = abs_fp
+                        matched_new.add(abs_fp)
+                        moved.append((old_path, abs_fp))
+                        logger.info(
+                            "Detected moved file: %s -> %s (doc_id %s -> %s)",
+                            old_path, abs_fp, moved_doc_id, new_doc_id,
+                        )
+                    else:
+                        logger.debug(
+                            "Move-detection skipped for %s -> %s "
+                            "(doc_id collision); will full re-index",
+                            old_path, abs_fp,
+                        )
+
+    # --- Pass 2: size-based heuristic for stat-mode moves ---
+    # Collect orphan candidates: indexed paths whose files disappeared from disk
+    # and were not already remapped by Pass 1.
+    orphan_by_key: dict[str, list[tuple[str, str]]] = {}  # "basename:size" → [(path, hash)]
+    for sp, sh in list(all_meta.items()):
+        if sp in expanded_abs or os.path.isfile(sp):
+            continue
+        size_str = _extract_size_from_fingerprint(sh)
+        if not size_str:
+            continue
+        bn = os.path.basename(sp)
+        orphan_by_key.setdefault(f"{bn}:{size_str}", []).append((sp, sh))
+
+    if orphan_by_key:
+        for fp in expanded:
+            abs_fp = os.path.abspath(fp)
+            if abs_fp in all_meta or abs_fp in matched_new:
+                continue
+            try:
+                st = os.stat(abs_fp)
+            except OSError:
+                continue
+            bn = os.path.basename(abs_fp)
+            key = f"{bn}:{st.st_size}"
+            candidates = orphan_by_key.get(key, [])
+            for old_path, stored_hash in candidates:
+                if old_path not in all_meta:
+                    continue  # already claimed by another new file
+                moved_doc_id = fts.get_doc_id_by_source_path(old_path)
+                if not moved_doc_id:
+                    continue
+                new_doc_id = fp_to_doc_id.get(fp, os.path.splitext(bn)[0])
+                if fts.rename_document(moved_doc_id, new_doc_id, new_doc_id, abs_fp):
+                    new_fh = file_hash_fn(abs_fp)
+                    fts.set_index_meta(abs_fp, new_fh)
+                    all_meta.pop(old_path, None)
+                    all_meta[abs_fp] = new_fh
+                    moved.append((old_path, abs_fp))
+                    logger.info(
+                        "Detected moved/renamed file (size heuristic): %s -> %s",
+                        old_path, abs_fp,
+                    )
+                    break
+
+    return moved
+
+
+async def build_index(
+    paths: list[str],
+    output_dir: str = "./indexes",
+    *,
+    db_path: str = "",
+    if_add_node_summary: Optional[bool] = None,
+    if_add_doc_description: Optional[bool] = None,
+    if_add_node_text: Optional[bool] = None,
+    if_add_node_id: Optional[bool] = None,
+    max_concurrency: Optional[int] = None,
+    force: bool = False,
+    ignore_dirs: frozenset[str] = DEFAULT_IGNORE_DIRS,
+    respect_gitignore: bool = True,
+    max_files: int = MAX_DIR_FILES,
+    prune: Optional[bool] = None,
+    progress_callback: Optional[callable] = None,
+    sub_progress_callback: Optional[callable] = None,
+    **kwargs,
+) -> list[Document]:
+    """
+    Build tree indexes for multiple files. Returns list of Document objects ready for search.
+
+    All parameters default to ``get_config()`` values when not explicitly set.
+
+    Args:
+        paths: list of file paths, glob patterns, or directories
+            (e.g. ``["docs/*.md", "paper.txt", "src/"]``)
+        output_dir: directory for the database file (used to derive db_path if db_path is empty)
+        db_path: path to the SQLite database file. If empty, defaults to ``{output_dir}/index.db``.
+        max_concurrency: max concurrent indexing tasks
+        force: force re-index even if file unchanged (default: False)
+        ignore_dirs: directory names to skip during recursive walk
+        respect_gitignore: honour ``.gitignore`` files when walking directories
+        max_files: safety cap on files discovered per directory walk
+        prune: if True, delete documents whose source files are no longer
+            reachable through ``paths`` (orphan cleanup). When ``None``,
+            defaults to True iff at least one entry of ``paths`` is a directory
+            or recursive glob (full-scope reindex), False otherwise.
+        **kwargs: passed through to individual parsers
+
+    Returns:
+        list of Document objects (directly usable with search())
+    """
+    from .config import get_config
+    from .fts import FTS5Index
+    cfg = get_config()
+
+    # Resolve defaults from config
+    if if_add_node_summary is None:
+        if_add_node_summary = cfg.if_add_node_summary
+    if if_add_doc_description is None:
+        if_add_doc_description = cfg.if_add_doc_description
+    if if_add_node_text is None:
+        if_add_node_text = cfg.if_add_node_text
+    if if_add_node_id is None:
+        if_add_node_id = cfg.if_add_node_id
+    if max_concurrency is None:
+        max_concurrency = cfg.max_concurrency
+
+    # Resolve db_path
+    if not db_path:
+        db_path = os.path.join(output_dir, "index.db")
+    os.makedirs(os.path.dirname(os.path.abspath(db_path)), exist_ok=True)
+
+    # 图片落盘存储（与 index.db 同目录的 images/ 子目录）
+    from .parsers.image_store import ImageStore
+    images_root = Path(db_path).parent / "images"
+    image_store = ImageStore(images_root)
+    # PST 附件落盘存储（与 index.db 同目录的 pst_attachments/ 子目录，ADR-0005）
+    from .parsers.pst_attachment_store import PstAttachmentStore
+    pst_att_store = PstAttachmentStore(Path(db_path).parent / "pst_attachments")
+    if force:
+        image_store.purge_all()
+        pst_att_store.purge_all()
+
+    # 计算 rel_path 用的 base 目录（paths 里第一个目录，即 search_path）
+    base_dir = ""
+    for p in paths:
+        if os.path.isdir(p):
+            base_dir = os.path.abspath(p)
+            break
+
+    # Expand globs, files, and directories via resolve_paths
+    expanded = resolve_paths(
+        paths,
+        ignore_dirs=ignore_dirs,
+        respect_gitignore=respect_gitignore,
+        max_files=max_files,
+    )
+    total_files = len(expanded)
+    processed_counter = [0]
+    _progress_lock = asyncio.Lock()
+    if not expanded:
+        raise FileNotFoundError(f"No files found for patterns: {paths}")
+
+    # Pre-compute deterministic doc_ids: basename + path hash (always, for determinism)
+    _base_dir = os.path.dirname(os.path.abspath(db_path)) if db_path else os.getcwd()
+
+    def _doc_id_for(fp):
+        base = os.path.splitext(os.path.basename(fp))[0]
+        rel = os.path.relpath(os.path.abspath(fp), _base_dir)
+        h = hashlib.md5(rel.encode()).hexdigest()[:8]
+        return f"{base}_{h}"
+
+    _fp_to_doc_id = {fp: _doc_id_for(fp) for fp in expanded}
+
+    # Resolve prune policy: full-scope walks default to pruning orphans.
+    explicit_prune = prune is not None
+    if prune is None:
+        full_scope = any(
+            os.path.isdir(p) or "**" in p
+            for p in paths
+        )
+        prune = full_scope and cfg.prune_orphans_on_directory
+
+    # Open DB with advisory file lock so concurrent build_index() calls on the
+    # same DB serialize cleanly instead of racing on writes.
+    fts = FTS5Index(db_path=db_path, tokenize_log_path=os.path.join(os.path.dirname(db_path), "tokenize.log"))
+    _lock_handle = _acquire_index_lock(db_path)
+
+    # 如果锁获取失败（_NullLock），说明有其他进程在索引，跳过本次
+    if isinstance(_lock_handle, _NullLock):
+        logger.warning("Index is locked by another process, skipping this indexing run")
+        return []
+
+    # Incremental indexing: batch check file hashes via DB
+    to_index = []
+    skipped = []
+    file_hashes = {}
+    pruned_paths: list[str] = []
+
+    if force:
+        # Full rebuild: clear all failed file records
+        fts.clear_all_failed_files()
+        # 视觉解析队列一并清空（重建过程会重新登记）
+        fts.vision_clear()
+
+    if not force:
+        # Batch fetch all stored hashes in one query (instead of N queries)
+        all_meta = fts.get_all_index_meta()
+    elif cfg.allowed_source_types:
+        # force=True with source_type filter: still need all_meta for orphan cleanup
+        all_meta = fts.get_all_index_meta()
+    else:
+        all_meta = {}
+
+    # Pre-pass: detect moves/renames and remap source_path BEFORE pruning,
+    # so the orphan cleanup below doesn't drop a doc whose file just moved.
+    if not force:
+        _detect_and_apply_moves(
+            expanded, all_meta, fts, _fp_to_doc_id, file_hash_with_salts
+        )
+
+    # Orphan cleanup: docs in DB whose source_path is not in the new scope.
+    # Implicit prune (defaulted from a directory walk): only drop docs whose
+    # files are also gone from disk — preserves unrelated files for partial
+    # reindexes.
+    # Explicit `prune=True`: reduce the index to exactly `paths`.
+    if prune and all_meta:
+        expanded_abs = {os.path.abspath(p) for p in expanded}
+        # Pre-compute allowed extensions from source_type filter (if any)
+        source_type_exts = None
+        if cfg.allowed_source_types:
+            from .pathutil import get_allowed_extensions_for_source_types
+            source_type_exts = get_allowed_extensions_for_source_types(cfg.allowed_source_types)
+        prune_doc_ids: list[str] = []
+        for stored_path in list(all_meta.keys()):
+            if stored_path in expanded_abs:
+                continue
+            if not explicit_prune and os.path.isfile(stored_path):
+                # File exists on disk but was filtered out — only keep it if
+                # it would have been included WITHOUT the source_type filter.
+                if source_type_exts is not None:
+                    ext = os.path.splitext(stored_path)[1].lower()
+                    if ext not in source_type_exts:
+                        pass  # excluded by source_type → prune
+                    else:
+                        continue
+                else:
+                    continue
+            doc_id = fts.get_doc_id_by_source_path(stored_path)
+            # 多文档来源（PST 派生 "<file>#<entry_id>"）：级联删除派生文档
+            derived_ids = fts.get_doc_ids_by_source_prefix(stored_path + "#")
+            ids = ([doc_id] if doc_id else []) + derived_ids
+            if ids:
+                prune_doc_ids.extend(ids)
+                pruned_paths.append(stored_path)
+                all_meta.pop(stored_path, None)
+        if prune_doc_ids:
+            fts.delete_documents(prune_doc_ids)
+            logger.info("Pruned %d orphan document(s) from index", len(prune_doc_ids))
+        # 被 prune 的源文件同步移出视觉解析队列
+        for stored_path in pruned_paths:
+            fts.vision_remove(stored_path)
+
+        # Clean up shadow MD files for pruned binary sources
+        from .parsers.registry import is_binary_extension
+        for pruned_path in pruned_paths:
+            ext = os.path.splitext(pruned_path)[1].lower()
+            if is_binary_extension(ext):
+                md_path = shadow_md_path(pruned_path)
+                if os.path.exists(md_path):
+                    try:
+                        os.remove(md_path)
+                        logger.debug("Removed orphan shadow MD: %s", md_path)
+                    except OSError as e:
+                        logger.debug("Failed to remove shadow MD %s: %s", md_path, e)
+                # 清理被删文档的图片
+                if base_dir:
+                    pruned_rel = os.path.relpath(pruned_path, base_dir).replace(os.sep, "/")
+                    image_store.purge_doc(pruned_rel)
+                # 清理被删 PST 的落盘附件（ADR-0005）
+                if ext == ".pst" and base_dir:
+                    pruned_rel = os.path.relpath(pruned_path, base_dir).replace(os.sep, "/")
+                    pst_att_store.purge_doc(pruned_rel)
+
+    for fp in expanded:
+        abs_fp = os.path.abspath(fp)
+        # 指纹含解析器格式盐（PST：ADR-0005 输出格式变化时自动重建）
+        fh = file_hash_with_salts(abs_fp)
+        if not fh:
+            # File disappeared after glob expansion (e.g. broken symlink)
+            logger.debug("Skipping missing file: %s", abs_fp)
+            continue
+        file_hashes[abs_fp] = fh
+        if not force:
+            stored_hash = all_meta.get(abs_fp)
+            if stored_hash == fh:
+                # source_path lookup catches both same-name and moved files.
+                # Multi-doc sources (e.g. PST: docs keyed "<file>#<entry_id>")
+                # have no doc at the exact path — check the derived prefix.
+                if fts.get_doc_id_by_source_path(abs_fp) is not None \
+                        or fts.has_docs_with_source_prefix(abs_fp + "#"):
+                    skipped.append(fp)
+                    processed_counter[0] += 1
+                    if progress_callback:
+                        progress_callback(fp, processed_counter[0], total_files)
+                    continue
+        to_index.append(fp)
+
+    if skipped:
+        logger.info("Skipped %d unchanged file(s)", len(skipped))
+
+    # Filter out files that have exceeded the consecutive failure threshold
+    excluded_count = 0
+    if not force and to_index:
+        failed_records = fts.get_all_failed_files()  # {path: (fail_count, file_hash, last_error)}
+        max_fail_count = cfg.max_index_fail_count
+        excluded = []
+        remaining = []
+        for fp in to_index:
+            abs_fp = os.path.abspath(fp)
+            record = failed_records.get(abs_fp)
+            if record and record[0] >= max_fail_count:
+                # Check if file content has changed since last failure
+                current_hash = file_hashes.get(abs_fp, "")
+                if record[1] and record[1] == current_hash:
+                    excluded.append(fp)
+                    continue
+            remaining.append(fp)
+        to_index = remaining
+        if excluded:
+            excluded_count = len(excluded)
+            logger.warning("Skipped %d file(s) with %d+ consecutive failures", excluded_count, max_fail_count)
+            for fp in excluded:
+                abs_fp = os.path.abspath(fp)
+                record = failed_records.get(abs_fp)
+                last_error = record[2] if record else ""
+                if last_error:
+                    logger.warning("  Skipped %s: %s", fp, last_error)
+                else:
+                    logger.warning("  Skipped %s: unknown error", fp)
+                processed_counter[0] += 1
+                if progress_callback:
+                    progress_callback(fp, processed_counter[0], total_files)
+
+    logger.info("Building indexes for %d file(s) (concurrency=%d)...", len(to_index), max_concurrency)
+
+    build_start = time.monotonic()
+    semaphore = asyncio.Semaphore(max_concurrency)
+    # PST 专用并发上限：PST 解析启 sidecar 子进程 + 加载 GB 级文件 + 全量附件提取，
+    # 多个并发会耗尽内存/CPU/IO 触发 sidecar 崩溃（exit 0xC000013A）。单独低并发限流，
+    # 同时缓解"极度缓慢"和"崩溃"两个根因。
+    pst_semaphore = asyncio.Semaphore(getattr(cfg, "max_pst_concurrency", 1))
+    # Collect per-file timing and source_type for stats
+    _file_timings: dict[str, tuple[str, float]] = {}  # fp -> (source_type, elapsed_s)
+    _failed_paths: list[str] = []
+
+    # Progress bar for parsing stage
+    _parse_bar = tqdm(total=len(to_index), desc="Parsing", unit="file",
+                      dynamic_ncols=True, disable=not to_index)
+
+    async def _index_one(fp: str) -> dict | None:
+        ext = os.path.splitext(fp)[1].lower()
+        # PST 重量级（sidecar 子进程 + GB 级文件 + 全量附件提取）：先过 pst_semaphore
+        # 限流（等待时不占通用 semaphore 槽），再过通用并发信号量。
+        pst_gate = pst_semaphore if ext == ".pst" else nullcontext()
+        async with pst_gate, semaphore:
+            fname = os.path.basename(fp)
+            _parse_bar.set_postfix_str(fname, refresh=False)
+            t0 = time.monotonic()
+            try:
+                rel_path = ""
+                if base_dir:
+                    rel_path = os.path.relpath(fp, base_dir).replace(os.sep, "/")
+                # 该文件重抽前先清掉它的旧图（幂等；force 模式已 purge_all，此处为空操作）
+                if rel_path:
+                    image_store.purge_doc(rel_path)
+                    # PST 重索引：清掉旧落盘附件（幂等，重新提取）
+                    if ext == ".pst":
+                        pst_att_store.purge_doc(rel_path)
+                common = dict(
+                    if_add_node_summary=if_add_node_summary,
+                    if_add_doc_description=if_add_doc_description,
+                    if_add_node_text=if_add_node_text,
+                    if_add_node_id=if_add_node_id,
+                    image_store=image_store,
+                    rel_path=rel_path,
+                    pst_attachment_store=pst_att_store,
+                    sub_progress_callback=(
+                        (lambda n: sub_progress_callback(fp, n))
+                        if sub_progress_callback else None
+                    ),
+                    **kwargs,
+                )
+
+                # Use ParserRegistry for dispatch (built-in parsers auto-registered)
+                from .parsers import get_parser, SOURCE_TYPE_MAP
+                parser_fn = get_parser(ext)
+                if parser_fn is not None:
+                    result = await parser_fn(fp, **common)
+                else:
+                    # Unknown extension: fall back to text_to_tree
+                    result = await text_to_tree(text_path=fp, **common)
+
+                # Tag source_type for search routing
+                source_type = SOURCE_TYPE_MAP.get(ext, "text")
+                result["source_type"] = source_type
+
+                # 图像文件：占位节点已进索引，登记到视觉解析队列（后台 worker 消费）
+                if source_type == "image" and result.get("vision_pending"):
+                    fts.vision_enqueue(os.path.abspath(fp), rel_path)
+
+                # Generate shadow MD for binary files (concurrent with parsing)
+                if cfg.enable_shadow_md:
+                    from .parsers.registry import is_binary_extension
+                    if is_binary_extension(ext):
+                        try:
+                            _generate_shadow_md(os.path.abspath(fp))
+                        except Exception as e:
+                            logger.debug("Shadow MD generation failed for %s: %s", fp, e)
+
+                _file_timings[fp] = (source_type, time.monotonic() - t0)
+                # Call progress callback if provided
+                async with _progress_lock:
+                    processed_counter[0] += 1
+                    if progress_callback:
+                        progress_callback(fp, processed_counter[0], total_files)
+                return result
+            except Exception as e:
+                logger.warning("Failed to index %s: %s", fp, e)
+                _failed_paths.append(fp)
+                abs_fp = os.path.abspath(fp)
+                fts.upsert_failed_file(abs_fp, str(e), file_hashes.get(abs_fp, ""))
+                _file_timings[fp] = ("(failed)", time.monotonic() - t0)
+                async with _progress_lock:
+                    processed_counter[0] += 1
+                    if progress_callback:
+                        progress_callback(fp, processed_counter[0], total_files)
+                return None
+            finally:
+                _parse_bar.update()
+
+    raw_results = await asyncio.gather(*(_index_one(fp) for fp in to_index))
+    _parse_bar.close()
+
+    # Save results to DB and collect Document objects
+    result_map = {fp: r for fp, r in zip(to_index, raw_results) if r is not None}
+    documents = []
+
+    # Batch load all skipped documents in one query (instead of N individual loads)
+    # Key by source_path (unique and stable) instead of doc_id (may change)
+    if skipped:
+        all_docs_from_db = {
+            d.metadata.get("source_path", ""): d
+            for d in fts.load_all_documents()
+            if d.metadata.get("source_path")
+        }
+    else:
+        all_docs_from_db = {}
+
+    # Progress bar for Indexing stage (batch commit every N files to reduce fsync)
+    _COMMIT_BATCH = 500
+    _has_work = bool(result_map)
+    _save_bar = tqdm(total=len(expanded), desc="Indexing", unit="file",
+                     dynamic_ncols=True, disable=not _has_work)
+    _pending_commits = 0
+    # Aggregate node-level diff stats across all reindexed docs.
+    diff_totals = {"added": 0, "changed": 0, "removed": 0, "kept": 0}
+    optimize_threshold = cfg.auto_optimize_threshold
+    docs_since_optimize = 0
+    for fp in expanded:
+        name = _fp_to_doc_id[fp]
+        if fp in result_map:
+            _save_bar.set_postfix_str(os.path.basename(fp), refresh=False)
+            result = result_map[fp]
+            abs_fp = os.path.abspath(fp)
+            file_h = file_hashes.get(abs_fp, "")
+
+            if result.get("multi_docs") is not None:
+                # 多文档来源（PST 等）：一个文件 → N 个派生文档。
+                # 派生 doc_id = <file_doc_id>__<entry>；source_path = <file>#<entry>。
+                # 已被移除的派生文档（邮件删除）按差集清除；其余走节点级增量。
+                trees = result["multi_docs"]
+                new_docs = []
+                for t in trees:
+                    sp = t.get("source_path", "")
+                    entry = sp.rsplit("#", 1)[-1] if "#" in sp else str(len(new_docs))
+                    new_docs.append(Document(
+                        doc_id=f"{name}__{entry}",
+                        doc_name=t.get("doc_name", name),
+                        structure=t.get("structure", []),
+                        doc_description=t.get("doc_description", ""),
+                        metadata={"source_path": sp},
+                        source_type=result.get("source_type", ""),
+                    ))
+                new_ids = {d.doc_id for d in new_docs}
+                old_ids = set(fts.get_doc_ids_by_source_prefix(abs_fp + "#"))
+                removed_ids = sorted(old_ids - new_ids)
+                if removed_ids:
+                    fts.delete_documents(removed_ids)
+                    # 被移除派生文档（邮件删除）的落盘附件级联清理（ADR-0005）
+                    if base_dir:
+                        pst_rel = os.path.relpath(abs_fp, base_dir).replace(os.sep, "/")
+                        for rid in removed_ids:
+                            entry = rid.rsplit("__", 1)[-1]
+                            pst_att_store.purge_email(pst_rel, entry)
+                    logger.info("Removed %d stale derived doc(s) for %s",
+                                len(removed_ids), fp)
+                for doc in new_docs:
+                    fts.index_document(doc, auto_commit=False)
+                    d = fts.last_node_diff
+                    for k in diff_totals:
+                        diff_totals[k] += d[k]
+                    _pending_commits += 1
+                    docs_since_optimize += 1
+                    if _pending_commits >= _COMMIT_BATCH:
+                        fts.commit()
+                        _pending_commits = 0
+                    if optimize_threshold and docs_since_optimize >= optimize_threshold:
+                        fts.optimize()
+                        docs_since_optimize = 0
+                # 邮件元数据入库（pst_email_meta 表，供列表分页查询，ADR-0005）
+                for t, doc in zip(trees, new_docs):
+                    meta = t.get("email_meta")
+                    if meta:
+                        fts.upsert_email_meta(doc.doc_id, abs_fp, meta)
+                # 文件指纹记在物理文件路径上（派生文档不写 index_meta）
+                fts.set_index_meta(abs_fp, file_h)
+                fts.clear_failed_file(abs_fp)
+                logger.debug("Indexed %d derived docs: %s -> %s",
+                             len(new_docs), fp, db_path)
+                documents.extend(new_docs)
+                _save_bar.update()
+                continue
+
+            doc = Document(
+                doc_id=name,
+                doc_name=result.get("doc_name", name),
+                structure=result.get("structure", []),
+                doc_description=result.get("doc_description", ""),
+                metadata={"source_path": result.get("source_path", "")},
+                source_type=result.get("source_type", ""),
+            )
+            # index_document writes nodes, fts_nodes, documents AND index_meta
+            # in a single atomic transaction (auto_commit handles batching).
+            fts.index_document(doc, auto_commit=False, file_hash=file_h)
+            # Clear any prior failure record for this file
+            fts.clear_failed_file(abs_fp)
+            d = fts.last_node_diff
+            for k in diff_totals:
+                diff_totals[k] += d[k]
+            _pending_commits += 1
+            docs_since_optimize += 1
+            if _pending_commits >= _COMMIT_BATCH:
+                fts.commit()
+                _pending_commits = 0
+            if optimize_threshold and docs_since_optimize >= optimize_threshold:
+                fts.optimize()
+                docs_since_optimize = 0
+            logger.debug("Indexed: %s -> %s (doc_id=%s)", fp, db_path, name)
+        else:
+            # Skipped file: use batch-loaded docs (lookup by source_path, not doc_id)
+            abs_fp = os.path.abspath(fp)
+            doc = all_docs_from_db.get(abs_fp)
+            if doc is None:
+                # 多文档来源（PST 等）：物理路径无精确匹配，按派生前缀收集
+                derived = [d for sp, d in all_docs_from_db.items()
+                           if sp.startswith(abs_fp + "#")]
+                if derived:
+                    documents.extend(derived)
+                    _save_bar.update()
+                    continue
+                logger.debug("Skipped file %s has no document in DB (excluded by failure threshold)", fp)
+                _save_bar.update()
+                continue
+        documents.append(doc)
+        _save_bar.update()
+    # Final commit for remaining pending writes
+    if _pending_commits > 0:
+        logger.info("Committing remaining %d documents to database...", _pending_commits)
+        fts.commit()
+    _save_bar.close()
+
+    # Fold WAL sidecar back into the main DB so long-running daemons don't
+    # accumulate a multi-GB -wal file across many incremental builds.
+    fts.wal_checkpoint("TRUNCATE")
+
+    # ---------------------------------------------------------------
+    # Build IndexStats
+    # ---------------------------------------------------------------
+    build_elapsed = time.monotonic() - build_start
+
+    # Count total nodes in newly indexed documents
+    total_nodes = 0
+    for doc in documents:
+        total_nodes += len(flatten_tree(doc.structure))
+
+    # Per source_type aggregation
+    per_type: dict[str, dict] = {}
+    for fp, (stype, elapsed) in _file_timings.items():
+        if stype == "(failed)":
+            continue
+        entry = per_type.setdefault(stype, {"count": 0, "nodes": 0, "time_s": 0.0})
+        entry["count"] += 1
+        entry["time_s"] += elapsed
+        # Count nodes for this file
+        result = result_map.get(fp)
+        if result:
+            if result.get("multi_docs") is not None:
+                entry["nodes"] += sum(
+                    len(flatten_tree(t.get("structure", [])))
+                    for t in result["multi_docs"]
+                )
+            else:
+                entry["nodes"] += len(flatten_tree(result.get("structure", [])))
+
+    # Database size
+    db_size = 0
+    if db_path and os.path.isfile(db_path):
+        try:
+            db_size = os.path.getsize(db_path)
+        except OSError:
+            pass
+
+    stats = IndexStats(
+        total_files=len(expanded),
+        indexed_files=len(to_index) - len(_failed_paths),
+        skipped_files=len(skipped),
+        failed_files=len(_failed_paths),
+        excluded_files=excluded_count,
+        total_nodes=total_nodes,
+        total_time_s=build_elapsed,
+        per_type=per_type,
+        db_path=db_path,
+        db_size_bytes=db_size,
+        failed_paths=_failed_paths,
+        node_diff=diff_totals,
+        pruned_paths=pruned_paths,
+    )
+
+    # Attach stats to the returned list for easy access
+    class _DocumentList(list):
+        """List subclass that carries IndexStats."""
+        stats: IndexStats = None  # type: ignore[assignment]
+
+    doc_list = _DocumentList(documents)
+    doc_list.stats = stats
+
+    fts.close()
+    _lock_handle.release()
+    return doc_list