seam-code 0.3.0__py3-none-any.whl

seam/indexer/sync.py

@@ -0,0 +1,287 @@
+"""Filesystem reconcile for Seam — the engine behind `seam sync`.
+
+Performs a mtime-pre-filter → SHA-1-confirm reconcile of the existing index
+against the current on-disk state, then runs a full cluster recompute gated
+on whether the graph actually changed.
+
+Import hierarchy:
+    cli.main → indexer.sync → indexer.pipeline, indexer.db, indexer.cluster_index
+    (never the reverse — this module must not import from cli)
+
+Design decisions:
+    - Reconcile is filesystem-based (mtime + SHA-1), NOT git-based.
+      Catches non-git repos AND committed changes from pull/checkout/merge.
+    - mtime pre-filter: if stored mtime == on-disk st_mtime → unchanged, no read.
+    - Hash confirm: if mtime differs but hash matches → unchanged, no re-index.
+      (touch-without-content-change pays one hash per sync, never a re-index.)
+    - Cluster recompute is FULL (not incremental) and gated on graph_changed.
+      One new edge can re-partition unrelated communities → incremental is wrong.
+    - Per-file errors are swallowed by index_one_file (returns None → skipped).
+      sync() itself never raises for per-file issues.
+    - Logging: INFO for the final summary line, DEBUG for per-file decisions.
+"""
+
+import logging
+import sqlite3
+from pathlib import Path
+from typing import TypedDict
+
+from seam.indexer.cluster_index import index_clusters
+from seam.indexer.db import delete_file
+from seam.indexer.pipeline import index_one_file, sha1, walk_project
+from seam.indexer.synthesis_index import index_synthesis
+
+logger = logging.getLogger(__name__)
+
+
+class SyncResult(TypedDict):
+    """Outcome of one sync() call.
+
+    Keys mirror the PRD §Implementation Decisions shape exactly.
+    cluster_count is None when clusters were NOT recomputed this sync.
+    synthesis_count is None when synthesis was NOT run this sync.
+    """
+
+    added: int              # files present on disk, absent from index → indexed
+    modified: int           # tracked files whose content hash changed → re-indexed
+    removed: int            # tracked files no longer on disk → deleted
+    unchanged: int          # tracked files skipped (mtime match or hash match)
+    skipped: int            # files index_one_file declined (unsupported/binary/error)
+    graph_changed: bool     # (added + modified + removed) > 0
+    clusters_recomputed: bool  # whether index_clusters ran this sync
+    cluster_count: int | None  # result of index_clusters when it ran, else None
+    synthesis_recomputed: bool  # whether index_synthesis ran this sync
+    synthesis_count: int | None  # result of index_synthesis when it ran, else None
+
+
+def _load_tracked(conn: sqlite3.Connection) -> dict[str, tuple[float, str]]:
+    """Load all tracked files from the DB as {abs_path: (mtime, file_hash)}.
+
+    WHY: one bulk read at reconcile start is cheaper than per-file queries.
+    Returns absolute string paths as keys so comparison with walk_project output
+    (which also returns absolute paths) is direct.
+
+    Excludes synthetic file rows (path starting with ':') used by post-passes
+    like index_synthesis to store non-file-scoped edges. These rows are not real
+    files on disk and must never be treated as "removed" by the reconcile loop.
+    """
+    rows = conn.execute(
+        "SELECT path, mtime, file_hash FROM files WHERE path NOT LIKE ':%'"
+    ).fetchall()
+    return {row["path"]: (float(row["mtime"]), row["file_hash"]) for row in rows}
+
+
+def sync(
+    conn: sqlite3.Connection,
+    root: Path,
+    *,
+    recompute_clusters: bool = True,
+    force_clusters: bool = False,
+    naming_mode: str,
+    llm_api_key: str | None,
+    llm_model: str | None,
+    min_size: int,
+    synthesis_enabled: bool = True,
+    force_synthesis: bool = False,
+    fanout_cap: int = 40,
+) -> SyncResult:
+    """Reconcile the existing index against the current on-disk state.
+
+    Algorithm (per PRD):
+      1. Load tracked {abs_path: (mtime, hash)} from files table.
+      2. walk_project(root) → current indexable files.
+      3. For each current file:
+           - Not in tracked → index_one_file → added (None → skipped).
+           - In tracked, st_mtime == stored mtime → unchanged (no read).
+           - In tracked, mtime differs, read+sha1 → if hash same → unchanged
+             (touched but not changed); if hash differs → index_one_file → modified
+             (None → skipped).
+      4. For each tracked path absent from current set → delete_file → removed.
+      5. Gate: graph_changed = (added + modified + removed) > 0.
+         Run index_clusters iff recompute_clusters and (graph_changed or force_clusters).
+
+    Per-file errors: index_one_file already returns None on any per-file failure;
+    sync counts them in `skipped` and continues. DB-level errors propagate to the CLI.
+
+    Args:
+        conn:               Open write-access SQLite connection.
+        root:               Project root to reconcile against.
+        recompute_clusters: Master switch — when False, clusters are never recomputed.
+        force_clusters:     Override gate — recompute clusters even when graph_changed is False.
+        naming_mode:        Passed verbatim to index_clusters (from config).
+        llm_api_key:        Passed verbatim to index_clusters (from config).
+        llm_model:          Passed verbatim to index_clusters (from config).
+        min_size:           Passed verbatim to index_clusters (from config).
+        synthesis_enabled:  When False, synthesis pass is skipped (SEAM_EDGE_SYNTHESIS=off).
+        force_synthesis:    Override gate — rerun synthesis even when graph_changed is False.
+        fanout_cap:         Per-channel fan-out cap for index_synthesis.
+
+    Returns:
+        SyncResult with counts and gate outcomes.
+    """
+    added = 0
+    modified = 0
+    removed = 0
+    unchanged = 0
+    skipped = 0
+
+    # ── Step 1: Load current DB state ─────────────────────────────────────────
+    # One bulk read is much cheaper than per-file queries in the reconcile loop.
+    tracked = _load_tracked(conn)
+
+    # ── Step 2: Walk on-disk files ────────────────────────────────────────────
+    current_paths = walk_project(root)
+    # Build a set of abs-string paths for fast "is this tracked?" lookup.
+    current_set = {str(p) for p in current_paths}
+
+    # ── Step 3: Reconcile each on-disk file ───────────────────────────────────
+    for path in current_paths:
+        abs_str = str(path)
+
+        if abs_str not in tracked:
+            # New file — not in index at all → index it.
+            logger.debug("sync: NEW %s", path)
+            result = index_one_file(conn, path)
+            if result is None:
+                skipped += 1
+                logger.debug("sync: SKIPPED (index failed) %s", path)
+            else:
+                added += 1
+        else:
+            stored_mtime, stored_hash = tracked[abs_str]
+
+            # Cheap mtime pre-filter: if mtime matches → definitely unchanged.
+            try:
+                disk_mtime = path.stat().st_mtime
+            except OSError as exc:
+                # File disappeared between walk and stat — treat as skipped.
+                # It will show as removed in step 4 on the next sync.
+                logger.debug("sync: stat failed for %s: %s — skipping", path, exc)
+                skipped += 1
+                continue
+
+            if disk_mtime == stored_mtime:
+                # mtime matches → unchanged without reading content.
+                logger.debug("sync: UNCHANGED (mtime match) %s", path)
+                unchanged += 1
+                continue
+
+            # mtime differs — need to read and hash to decide.
+            try:
+                content = path.read_bytes()
+            except OSError as exc:
+                logger.debug("sync: read failed for %s: %s — skipping", path, exc)
+                skipped += 1
+                continue
+
+            disk_hash = sha1(content)
+
+            if disk_hash == stored_hash:
+                # mtime changed but content is identical (e.g. `touch`).
+                # WHY: we do NOT update stored mtime here — that would cost
+                # a write for zero benefit. On the next sync this file re-hashes
+                # once, still no re-index. Documented accepted inefficiency.
+                logger.debug("sync: UNCHANGED (hash match, touch) %s", path)
+                unchanged += 1
+            else:
+                # Content actually changed → re-index.
+                logger.debug("sync: MODIFIED %s", path)
+                result = index_one_file(conn, path)
+                if result is None:
+                    skipped += 1
+                    logger.debug("sync: SKIPPED (re-index failed) %s", path)
+                else:
+                    modified += 1
+
+    # ── Step 4: Delete tracked files no longer on disk ────────────────────────
+    # Double-check the file is ACTUALLY gone before deleting (CodeGraph's
+    # existsSync guard, roadmap §6.1). A tracked path can be absent from the
+    # walk set for benign reasons — a transient FS/permission hiccup, a
+    # wrong-directory sync, or a --db-dir pointed at another project's index.
+    # Trusting the walk set alone would let any of those silently wipe the
+    # entire index. We only remove a file once it genuinely no longer exists.
+    for abs_str in tracked:
+        if abs_str not in current_set and not Path(abs_str).exists():
+            logger.debug("sync: REMOVED %s", abs_str)
+            delete_file(conn, Path(abs_str))
+            removed += 1
+
+    # ── Step 5: Gate cluster recompute ────────────────────────────────────────
+    graph_changed = (added + modified + removed) > 0
+
+    clusters_recomputed = False
+    cluster_count: int | None = None
+
+    if recompute_clusters and (graph_changed or force_clusters):
+        logger.debug(
+            "sync: running index_clusters (graph_changed=%s, force=%s)",
+            graph_changed,
+            force_clusters,
+        )
+        cluster_count = index_clusters(
+            conn,
+            naming_mode=naming_mode,
+            llm_api_key=llm_api_key,
+            llm_model=llm_model,
+            min_size=min_size,
+        )
+        # index_clusters returns -1 on failure (it never raises). "Recomputed"
+        # must mean "clusters were successfully refreshed" — a failed pass leaves
+        # them stale, so do NOT claim success. The -1 sentinel is preserved in
+        # cluster_count so callers can tell "ran but failed" (-1) apart from
+        # "did not run" (None), and the CLI surfaces it as a visible failure
+        # (mirroring `seam init`'s clustering_failed guard).
+        clusters_recomputed = cluster_count >= 0
+
+    # ── Step 6: Gate synthesis recompute ──────────────────────────────────────
+    # Run synthesis iff synthesis_enabled AND (graph_changed OR force_synthesis).
+    # Mirrors the cluster-recompute gate exactly: a full-graph pass is needed
+    # whenever the edge graph has changed (a new interface or new implementation
+    # may affect the override fan-out). --force-synthesis handles the watcher-
+    # already-indexed-edits case (same as --force-clusters).
+    synthesis_recomputed = False
+    synthesis_count: int | None = None
+
+    if synthesis_enabled and (graph_changed or force_synthesis):
+        logger.debug(
+            "sync: running index_synthesis (graph_changed=%s, force=%s)",
+            graph_changed,
+            force_synthesis,
+        )
+        synthesis_count = index_synthesis(
+            conn,
+            enabled=synthesis_enabled,
+            fanout_cap=fanout_cap,
+        )
+        # index_synthesis returns -1 on failure (it never raises), 0 when disabled.
+        # synthesis_recomputed=True only when the pass actually succeeded (>=0).
+        synthesis_recomputed = synthesis_count >= 0
+
+    logger.info(
+        "sync: added=%d modified=%d removed=%d unchanged=%d skipped=%d "
+        "graph_changed=%s clusters_recomputed=%s cluster_count=%s "
+        "synthesis_recomputed=%s synthesis_count=%s",
+        added,
+        modified,
+        removed,
+        unchanged,
+        skipped,
+        graph_changed,
+        clusters_recomputed,
+        cluster_count,
+        synthesis_recomputed,
+        synthesis_count,
+    )
+
+    return SyncResult(
+        added=added,
+        modified=modified,
+        removed=removed,
+        unchanged=unchanged,
+        skipped=skipped,
+        graph_changed=graph_changed,
+        clusters_recomputed=clusters_recomputed,
+        cluster_count=cluster_count,
+        synthesis_recomputed=synthesis_recomputed,
+        synthesis_count=synthesis_count,
+    )

seam/indexer/synthesis_index.py

@@ -0,0 +1,291 @@
+"""Edge-synthesis orchestration — bridge between pure engine and DB persistence.
+
+Reads symbols+edges from the DB, calls synthesize_edges() (pure engine), and writes
+the synthesized edges in a single transaction.
+
+This module sits in the indexer layer and is called by cli/main.py after clustering
+(never per-file, always whole-graph). It mirrors the pattern of cluster_index.py
+exactly: a public never-raising wrapper around an inner implementation that may raise.
+
+Import hierarchy (enforced):
+    indexer/synthesis_index → analysis.synthesis + config
+    analysis modules → pure, no DB writes
+    cli → this module (not the other way)
+
+Design decisions:
+  - Reads all symbols + edges from the connection in one pass.
+  - Calls synthesize_edges (pure) then writes in ONE transaction.
+  - Synthesized edges are stored under a special synthetic "file" row (':synthesis:')
+    that is NOT a real file on disk. This avoids relaxing the edges FK constraint while
+    keeping edges.file_id a valid FK reference. The synthetic row is upserted once and
+    is never deleted by the normal file-cascade mechanism (its path is ':synthesis:').
+  - ON EACH CALL: deletes ALL existing synthesized edges first (WHERE synthesized_by
+    IS NOT NULL), then re-inserts fresh ones. This ensures idempotency: calling
+    index_synthesis twice yields the same set of edges without duplicates.
+  - NEVER raises: returns -1 on error (signals failure to CLI); any error logs
+    a warning. Returns 0 when enabled=False. Returns count >= 0 on success.
+  - SEAM_EDGE_SYNTHESIS=off: returns 0 immediately, writes nothing.
+  - Watcher does NOT call this — only seam init and seam sync do.
+"""
+
+import logging
+import sqlite3
+from pathlib import Path
+
+from seam.analysis.synthesis import synthesize_edges
+
+logger = logging.getLogger(__name__)
+
+# Sentinel file path for synthesized edges — not a real on-disk file.
+# Using a colon-prefixed path to ensure it can never collide with a real repo path.
+_SYNTHESIS_FILE_PATH = ":synthesis:"
+
+
+def _ensure_synthesis_file_row(conn: sqlite3.Connection) -> int:
+    """Upsert the synthetic file row and return its id.
+
+    Synthesized edges are not file-scoped, but edges.file_id has a FK to files(id).
+    We use a permanent synthetic file row (path=':synthesis:') that is never
+    cascade-deleted by normal re-indexing. This preserves FK integrity without
+    relaxing constraints or adding nullable columns.
+
+    Returns the file_id of the synthetic row.
+    """
+    conn.execute(
+        """
+        INSERT INTO files (path, language, file_hash, mtime, indexed_at)
+        VALUES (?, '', '', 0.0, 0.0)
+        ON CONFLICT(path) DO NOTHING
+        """,
+        (_SYNTHESIS_FILE_PATH,),
+    )
+    row = conn.execute(
+        "SELECT id FROM files WHERE path = ?", (_SYNTHESIS_FILE_PATH,)
+    ).fetchone()
+    return row[0]
+
+
+def index_synthesis(
+    conn: sqlite3.Connection,
+    *,
+    enabled: bool,
+    fanout_cap: int,
+) -> int:
+    """Synthesize dynamic-dispatch edges and persist them to the DB.
+
+    Reads the full symbol+edge graph, calls the synthesis engine, and stores
+    the resulting synthesized edges in a single transaction.
+
+    Called by `seam init` after clustering and by `seam sync` when the graph
+    changed (or when --force-synthesis is passed). NEVER called by the watcher.
+
+    Args:
+        conn:        Open SQLite connection (must have write access).
+        enabled:     When False, skip synthesis entirely and return 0 (SEAM_EDGE_SYNTHESIS=off).
+        fanout_cap:  Per-channel fan-out cap passed to synthesize_edges (from config).
+
+    Returns:
+        Number of synthesized edges written (>= 0). Returns -1 on error (never raises).
+        Returns 0 when enabled=False.
+
+    WHY -1 (not 0) on error: lets the CLI distinguish "zero synthesized edges because
+    no interface-override patterns exist" from "synthesis failed." Same contract as
+    index_clusters (which also returns -1 on failure, never raises).
+    """
+    if not enabled:
+        logger.debug("synthesis_index: SEAM_EDGE_SYNTHESIS=off — skipping synthesis pass")
+        return 0
+
+    try:
+        return _index_synthesis_impl(conn, fanout_cap)
+    except Exception as exc:  # noqa: BLE001
+        logger.warning(
+            "synthesis_index: failed to synthesize edges (%s: %s) — "
+            "synthesized edges will be absent; run 'seam init' again to retry",
+            type(exc).__name__,
+            exc,
+        )
+        return -1
+
+
+def _load_file_sources(conn: sqlite3.Connection) -> dict[str, str]:
+    """Load source text for all real indexed files.
+
+    Reads the 'files' table for all non-synthetic paths, then reads each file from
+    disk. Silently skips files that are missing, unreadable, or exceed the size limit.
+
+    WHY here and not in the engine: the synthesis engine is a pure leaf (no DB/IO);
+    IO must happen in the bridge layer. We pass the loaded sources to synthesize_edges().
+
+    Returns a dict mapping file path string → source text (possibly empty if all failed).
+    Never raises.
+    """
+    from seam import config as cfg  # lazy import to keep leaf contract on synthesis.py
+
+    sources: dict[str, str] = {}
+    # Total-corpus budget: stop loading once cumulative source size crosses the cap so
+    # a huge monorepo cannot OOM the final init step. 0 = unlimited (see config knob).
+    max_total = cfg.SEAM_SYNTHESIS_MAX_SOURCE_BYTES
+    total_bytes = 0
+    loaded = 0
+    tracked = 0
+    capped = False
+    try:
+        path_rows = conn.execute(
+            "SELECT path FROM files WHERE path != ? ORDER BY path",
+            (_SYNTHESIS_FILE_PATH,),
+        ).fetchall()
+        tracked = len(path_rows)
+
+        for row in path_rows:
+            raw_path = row[0] if isinstance(row, (list, tuple)) else row["path"]
+            if not raw_path or raw_path == _SYNTHESIS_FILE_PATH:
+                continue
+            try:
+                p = Path(raw_path)
+                if not p.exists() or not p.is_file():
+                    continue
+                # Respect the max file size limit to avoid loading huge generated files.
+                size = p.stat().st_size
+                if size > cfg.SEAM_MAX_FILE_BYTES:
+                    continue
+                # Stop before crossing the total-corpus budget (bounded memory).
+                if max_total > 0 and total_bytes + size > max_total:
+                    capped = True
+                    break
+                # Read with errors="replace" to handle encoding issues gracefully.
+                sources[raw_path] = p.read_text(encoding="utf-8", errors="replace")
+                total_bytes += size
+                loaded += 1
+            except Exception:  # noqa: BLE001
+                # Missing / unreadable file — skip silently (never-raise contract).
+                continue
+
+        # Observability: never silently under-produce. A capped scan is a WARNING
+        # (synthesis will be partial); a complete scan logs a DEBUG summary so an
+        # operator can see how many of the tracked files actually fed the channels.
+        if capped:
+            logger.warning(
+                "synthesis_index: source-load budget reached (%d bytes, %d/%d files) — "
+                "source-text channels see a PARTIAL corpus; raise "
+                "SEAM_SYNTHESIS_MAX_SOURCE_BYTES to scan more",
+                total_bytes,
+                loaded,
+                tracked,
+            )
+        else:
+            logger.debug(
+                "synthesis_index: loaded %d/%d source files (%d bytes) for synthesis",
+                loaded,
+                tracked,
+                total_bytes,
+            )
+    except Exception as exc:  # noqa: BLE001
+        logger.warning(
+            "synthesis_index: failed to load file sources (%s: %s) — "
+            "source-text channels will have no input",
+            type(exc).__name__,
+            exc,
+        )
+    return sources
+
+
+def _index_synthesis_impl(conn: sqlite3.Connection, fanout_cap: int) -> int:
+    """Inner implementation. May raise — outer function is the guard.
+
+    WHY separate function: the outer wrapper catches ALL exceptions and converts
+    them to -1. Having a clean inner function makes the logic easier to reason
+    about and test (tests can call the inner function directly if needed).
+    """
+    # ── Step 1: Read all symbols ──────────────────────────────────────────────
+    # We need: name (for qualified method lookup), kind (to identify methods vs classes).
+    symbol_rows = conn.execute(
+        """
+        SELECT s.name, s.kind
+        FROM symbols s
+        ORDER BY s.name
+        """
+    ).fetchall()
+
+    if not symbol_rows:
+        logger.debug("synthesis_index: no symbols in index, skipping synthesis")
+        # Still delete stale synthesized edges (from a previous run when symbols existed).
+        with conn:
+            conn.execute("DELETE FROM edges WHERE synthesized_by IS NOT NULL")
+        return 0
+
+    symbols = [{"name": row["name"], "kind": row["kind"]} for row in symbol_rows]
+
+    # ── Step 2: Read all edges (only the ones needed for synthesis) ────────────
+    # We need: source, target, kind (to identify extends/implements pairs).
+    # Only read statically-extracted edges (synthesized_by IS NULL) to avoid
+    # feeding synthesized edges back into the engine (no feedback loop).
+    edge_rows = conn.execute(
+        """
+        SELECT DISTINCT source_name, target_name, kind
+        FROM edges
+        WHERE synthesized_by IS NULL
+        """
+    ).fetchall()
+
+    edges = [
+        {"source": row["source_name"], "target": row["target_name"], "kind": row["kind"]}
+        for row in edge_rows
+    ]
+
+    # ── Step 3: Load source text for source-text-based channels ─────────────
+    # The A1a/A1b channels (closure-collection, event-emitter) need the actual
+    # source text to scan for dispatch patterns. We read from the 'files' table —
+    # each row has a 'path' column with the on-disk path. We skip missing/unreadable
+    # files silently (never-raise contract), and skip the synthetic ':synthesis:' row.
+    file_sources: dict[str, str] = _load_file_sources(conn)
+
+    # ── Step 4: Run the pure synthesis engine ────────────────────────────────
+    # synthesize_edges is pure and never raises (it degrades to [] on error).
+    synth_edges = synthesize_edges(
+        symbols,
+        edges,
+        file_sources=file_sources,
+        fanout_cap=fanout_cap,
+    )
+
+    # ── Step 5: Write in ONE transaction ─────────────────────────────────────
+    # Upsert the synthetic file row, DELETE all previous synthesized edges, then
+    # INSERT the fresh ones — atomically. Folding the synthetic-row upsert in here
+    # (rather than a separate earlier transaction) keeps the whole write a single
+    # consistent unit: a crash can't leave the ':synthesis:' row with no edges.
+    # Idempotent: the DELETE makes a second call produce the same result.
+    with conn:
+        synth_file_id = _ensure_synthesis_file_row(conn)
+        # Clear all previous synthesized edges (from ALL channels) so this pass
+        # is idempotent: running index_synthesis twice produces the same result.
+        conn.execute("DELETE FROM edges WHERE synthesized_by IS NOT NULL")
+
+        if synth_edges:
+            conn.executemany(
+                """
+                INSERT INTO edges (source_name, target_name, kind, file_id, line, confidence, synthesized_by)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                """,
+                [
+                    (
+                        e["source"],
+                        e["target"],
+                        e["kind"],
+                        synth_file_id,
+                        e.get("line", 0),
+                        e["confidence"],
+                        e["synthesized_by"],
+                    )
+                    for e in synth_edges
+                ],
+            )
+
+    count = len(synth_edges)
+    logger.info(
+        "synthesis_index: wrote %d synthesized edges (%s file_id=%d)",
+        count,
+        _SYNTHESIS_FILE_PATH,
+        synth_file_id,
+    )
+    return count

seam/indexer/tokenize.py

@@ -0,0 +1,79 @@
+"""Identifier compound-split tokenization (Tier D #12) — search-recall leaf.
+
+PROBLEM: ``symbols_fts`` uses FTS5's default unicode61 tokenizer, which does NOT split
+camelCase or snake_case. So ``GlobalPushToTalkShortcutMonitor`` is one opaque token and a
+natural-language query ("push to talk shortcut monitor") can never reach its sub-words.
+The split must happen at INDEX time (a stored concatenated token can't be un-joined at
+query time), so the indexer writes ``split_identifier(name)`` into ``symbols.search_text``
+(a dedicated 4th FTS column) and the query layer splits query terms with the SAME function.
+
+LAYER: pure leaf — imports only stdlib. No DB, no config, no other seam modules. Used by
+both ``seam/indexer/db.py`` (index side) and ``seam/query/fts.py`` (query side) so the two
+sides tokenize identically. Every function is pure, deterministic, and never raises.
+
+WHY identifier-only (no docstring): the docstring is already its own ``symbols_fts`` column,
+so folding it into search_text would double-index prose and dilute ranking. search_text is
+kept to identifier vocabulary (name + qualified_name segments) only.
+"""
+
+import re
+
+# camelCase / acronym boundary insertion. Two passes, applied in order:
+#   1. "<any><Upper><lower>+"  → split before an Upper that starts a Word (handles the
+#      acronym→Word boundary: "HTTPServer" → "HTTP Server", "parseJSONData" → "...JSON Data").
+#   2. "<lower|digit><Upper>"  → split a lowercase/digit run before the next Upper
+#      ("fooBar" → "foo Bar", "v2Loader" → "v2 Loader").
+_BOUNDARY_ACRONYM_WORD = re.compile(r"(.)([A-Z][a-z]+)")
+_BOUNDARY_LOWER_UPPER = re.compile(r"([a-z0-9])([A-Z])")
+
+# Separators that delimit identifier parts across languages: dot (qualified names),
+# underscore (snake_case), hyphen (kebab), slash/colon (paths/namespaces).
+_SEPARATORS = re.compile(r"[._\-/:]+")
+
+
+def split_identifier(text: str) -> list[str]:
+    """Split a code identifier into lowercased sub-word tokens (deduped, order-preserving).
+
+    Examples:
+        "GlobalPushToTalkShortcutMonitor" -> ["global","push","to","talk","shortcut","monitor"]
+        "parseJSONData"                   -> ["parse","json","data"]
+        "find_cycle"                      -> ["find","cycle"]
+        "Class.method"                    -> ["class","method"]
+        "v2Loader"                        -> ["v2","loader"]
+        ""                                -> []
+
+    Pure and total: any string in, a (possibly empty) list of lowercase tokens out. Never raises.
+    """
+    if not text:
+        return []
+    # Normalize cross-language separators to spaces first, then insert camelCase boundaries.
+    s = _SEPARATORS.sub(" ", text)
+    s = _BOUNDARY_ACRONYM_WORD.sub(r"\1 \2", s)
+    s = _BOUNDARY_LOWER_UPPER.sub(r"\1 \2", s)
+
+    seen: set[str] = set()
+    out: list[str] = []
+    for tok in s.split():
+        low = tok.lower()
+        if low and low not in seen:
+            seen.add(low)
+            out.append(low)
+    return out
+
+
+def build_search_text(name: str, qualified_name: str | None = None) -> str:
+    """Return the space-joined search_text for a symbol: split(name) ∪ split(qualified_name).
+
+    qualified_name is folded in (deduped against the name tokens) to recover the enclosing
+    type/namespace word WHERE qualified_name carries it (e.g. 'Class.method' → adds 'class';
+    a bare top-level function whose qualified_name is just its own name adds nothing). docstring
+    is deliberately NOT folded — it is already its own FTS column. Never raises.
+    """
+    toks = split_identifier(name)
+    if qualified_name:
+        seen = set(toks)
+        for t in split_identifier(qualified_name):
+            if t not in seen:
+                seen.add(t)
+                toks.append(t)
+    return " ".join(toks)