scip-cli 2.7.0__tar.gz → 2.8.0__tar.gz

{scip_cli-2.7.0 → scip_cli-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: scip-cli
-Version: 2.7.0
+Version: 2.8.0
 Summary: Fast code intelligence via SCIP indexes
 Project-URL: Homepage, https://github.com/flesler/scip-cli
 Author: Ariel Flesler

scip_cli-2.8.0/docs/port-v28-board.md

@@ -0,0 +1,32 @@
+# v2.8.0 multi-port board
+
+**Goal:** Ship Python v2.8.0 (exclude globs), then port full v2.6.0→v2.8.0 gap to go/rust/zig in `/tmp`, direct-to-main, smoke-tested. Mirror GitHub releases optional.
+
+**Frozen reference:** `/tmp/scip-cli-2.8.0-freeze.patch` (pre-commit snapshot)
+
+## Gap to port (per mirror)
+
+| Release | Features | Manual port |
+|---|---|---|
+| v2.7.0 | `analyze --check`, `dead_files`, section FP warnings | analyze/*, commands/analyze, tests |
+| v2.8.0 | `reindex --exclude`, `excludeGlobs`, post-process prune | exclude, indexing, config, reindex, tests |
+
+`scripts/sync-upstream.sh --ref v2.8.0` covers SKILL, fixtures, templates only.
+
+## Lanes
+
+| Lane | Path | Status | Notes |
+|---|---|---|---|
+| P0 Python | `~/Code/scip-cli` | in progress | commit → v2.8.0 → push → release |
+| G1-go | `/tmp/scip-cli-go` | pending | direct-to-main |
+| G1-rust | `/tmp/scip-cli-rust` | pending | direct-to-main |
+| G1-zig | `/tmp/scip-cli-zig` | pending | direct-to-main |
+
+## Verify log
+
+| Check | Python | Go | Rust | Zig |
+|---|---|---|---|---|
+| scripts/test.sh | | | | |
+| exclude dogfood | | | | |
+| analyze --check | n/a | | | |
+| readonly /iterate review | | | | |

{scip_cli-2.7.0 → scip_cli-2.8.0}/scip_cli/SKILL.md

@@ -160,7 +160,9 @@ Directory detection uses the filesystem when present, otherwise an indexed path
 ### reindex
 
 ```bash
-reindex [--path DIR ...] [--tsconfig FILE_OR_GLOB ...] [--with-external]
+reindex [--path DIR ...] [--tsconfig FILE_OR_GLOB ...] [--exclude GLOB ...] [--with-external]
 ```
 
 `--path` and `--tsconfig` cannot be combined (**TypeScript only**). `--tsconfig` takes `tsconfig*.json` files (repeatable; globs expanded inside the tool). File-based runs default to one `scip-typescript` process per file so each gets its own heap (`SCIP_CLI_TS_INDEX_BATCH_SIZE` still overrides). Scope is saved as `index-scope.json` and reused until a full `reindex` with neither flag.
+
+`--exclude GLOB` omits matching files from the SQLite index after conversion (repeatable; merged with `excludeGlobs` in `.scip-cli.json`). Persisted in `index-exclude.json` until a full `reindex` with no `--exclude`. Patterns without `/` match basenames (`*.test.ts`); patterns with `/` match repo-relative paths (`tests/**`, `**/__tests__/**`). Indexers still parse excluded files when production code imports them — post-process removal is authoritative.

{scip_cli-2.7.0 → scip_cli-2.8.0}/scip_cli/__init__.py

@@ -1,3 +1,3 @@
 """scip-cli: Fast code intelligence via SCIP indexes."""
 
-__version__ = "2.7.0"
+__version__ = "2.8.0"

{scip_cli-2.7.0 → scip_cli-2.8.0}/scip_cli/__main__.py

@@ -166,6 +166,15 @@ def main() -> None:
             "Cannot be combined with --path."
         ),
     )
+    reindex_parser.add_argument(
+        "--exclude",
+        action="append",
+        metavar="GLOB",
+        help=(
+            "Omit indexed files matching GLOB from the SQLite index "
+            "(repeatable; merged with excludeGlobs in .scip-cli.json; persisted until full reindex)"
+        ),
+    )
     reindex_parser.add_argument(
         "--with-external",
         action="store_true",

{scip_cli-2.7.0 → scip_cli-2.8.0}/scip_cli/commands/reindex.py

@@ -9,6 +9,7 @@ from ..cache import (
     index_db_path,
     promote_next_index,
 )
+from ..exclude import save_persisted_exclude_globs
 from ..indexing import index_project, log_index_complete
 from ..paths import normalize_path_scope
 from ..project import Language, find_project_root_and_language
@@ -24,6 +25,7 @@ def main(args):
 
     path_args = getattr(args, "path", None) or []
     tsconfig_args = getattr(args, "tsconfig", None) or []
+    exclude_args = getattr(args, "exclude", None) or []
     if path_args and tsconfig_args:
         print("Error: reindex --path and --tsconfig cannot be combined", file=sys.stderr)
         sys.exit(1)
@@ -68,6 +70,12 @@ def main(args):
     else:
         save_index_scope(root, None)
 
+    if exclude_args:
+        save_persisted_exclude_globs(root, exclude_args)
+        print(f"Index exclude: {', '.join(exclude_args)}", file=sys.stderr)
+    else:
+        save_persisted_exclude_globs(root, None)
+
     cache_dir = get_cache_dir(root)
     cache_dir.mkdir(parents=True, exist_ok=True)
 

{scip_cli-2.7.0 → scip_cli-2.8.0}/scip_cli/config.py

@@ -16,6 +16,7 @@ class ProjectSettings:
     max_heap_mb: int | None = None
     index_roots: list[str] = field(default_factory=list)
     only_index_roots: bool = False
+    exclude_globs: list[str] = field(default_factory=list)
 
 
 def load_project_config(project_root: Path) -> ProjectSettings:
@@ -44,10 +45,15 @@ def load_project_config(project_root: Path) -> ProjectSettings:
     if type(only_index_roots) is not bool:
         raise RuntimeError(f"Invalid {CONFIG_FILENAME}: onlyIndexRoots must be a boolean")
 
+    exclude_globs = data.get("excludeGlobs", [])
+    if not isinstance(exclude_globs, list) or not all(isinstance(p, str) for p in exclude_globs):
+        raise RuntimeError(f"Invalid {CONFIG_FILENAME}: excludeGlobs must be a string array")
+
     return ProjectSettings(
         max_heap_mb=max_heap_mb,
         index_roots=index_roots,
         only_index_roots=only_index_roots,
+        exclude_globs=exclude_globs,
     )
 
 

scip_cli-2.8.0/scip_cli/exclude.py

@@ -0,0 +1,108 @@
+"""Persisted index exclude globs and path matching."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path, PurePosixPath
+
+from .config import load_project_config
+
+EXCLUDE_FILENAME = "index-exclude.json"
+
+
+def _exclude_path(project_root: Path) -> Path:
+    from .cache import get_cache_dir
+
+    return get_cache_dir(project_root) / EXCLUDE_FILENAME
+
+
+def load_persisted_exclude_globs(project_root: Path) -> tuple[str, ...]:
+    """Load exclude globs from the last scoped reindex --exclude, if any."""
+    path = _exclude_path(project_root)
+    if not path.is_file():
+        return ()
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return ()
+    if not isinstance(data, dict):
+        return ()
+    raw = data.get("globs")
+    if not raw or not isinstance(raw, list) or not all(isinstance(g, str) for g in raw):
+        return ()
+    return tuple(raw)
+
+
+def save_persisted_exclude_globs(project_root: Path, globs: list[str] | None) -> None:
+    """Persist or clear reindex --exclude globs for a project."""
+    path = _exclude_path(project_root)
+    if not globs:
+        if path.is_file():
+            path.unlink()
+        return
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps({"globs": globs}, indent=2) + "\n", encoding="utf-8")
+
+
+def resolve_exclude_globs(project_root: Path) -> tuple[str, ...]:
+    """Merge .scip-cli.json excludeGlobs with persisted reindex --exclude globs."""
+    settings = load_project_config(project_root)
+    merged: list[str] = []
+    seen: set[str] = set()
+    for glob in (*settings.exclude_globs, *load_persisted_exclude_globs(project_root)):
+        if glob and glob not in seen:
+            seen.add(glob)
+            merged.append(glob)
+    return tuple(merged)
+
+
+def _glob_to_regex(pattern: str) -> re.Pattern[str]:
+    """Translate a repo-relative glob to a regex (supports **, *, ?)."""
+    normalized = pattern.replace("\\", "/").lstrip("/")
+    parts: list[str] = []
+    index = 0
+    while index < len(normalized):
+        char = normalized[index]
+        if char == "*":
+            if index + 1 < len(normalized) and normalized[index + 1] == "*":
+                if index + 2 < len(normalized) and normalized[index + 2] == "/":
+                    parts.append("(?:.+/)?")
+                    index += 3
+                    continue
+                parts.append(".*")
+                index += 2
+                continue
+            parts.append("[^/]*")
+            index += 1
+            continue
+        if char == "?":
+            parts.append("[^/]")
+            index += 1
+            continue
+        if char in ".^$+{}|()[]":
+            parts.append(re.escape(char))
+        else:
+            parts.append(char)
+        index += 1
+    return re.compile("^" + "".join(parts) + "$")
+
+
+def path_matches_glob(relative_path: str, pattern: str) -> bool:
+    """Return True when a repo-relative path matches an exclude glob.
+
+    Patterns without '/' match the basename only (e.g. ``*.test.ts``).
+    Patterns with '/' match the full posix path (``tests/**``, ``src/*.spec.ts``).
+    """
+    path = relative_path.replace("\\", "/").lstrip("/")
+    glob = pattern.replace("\\", "/").lstrip("/")
+    if not glob:
+        return False
+    target = PurePosixPath(path).name if "/" not in glob else path
+    return bool(_glob_to_regex(glob).match(target))
+
+
+def path_matches_any_glob(relative_path: str, patterns: tuple[str, ...] | list[str]) -> bool:
+    if not patterns:
+        return False
+    return any(path_matches_glob(relative_path, pattern) for pattern in patterns)

{scip_cli-2.7.0 → scip_cli-2.8.0}/scip_cli/indexing/convert.py

@@ -68,7 +68,13 @@ def prefix_document_paths(db_path: Path, prefix: str) -> None:
         conn.close()
 
 
-def convert_scip_to_db(scip_path, db_path, *, document_path_prefix: Path | str | None = None):
+def convert_scip_to_db(
+    scip_path,
+    db_path,
+    *,
+    document_path_prefix: Path | str | None = None,
+    exclude_globs: tuple[str, ...] | None = None,
+):
     """Convert a SCIP protobuf file to a SQLite index at db_path."""
     db_path = Path(db_path)
     db_path.parent.mkdir(parents=True, exist_ok=True)
@@ -88,7 +94,7 @@ def convert_scip_to_db(scip_path, db_path, *, document_path_prefix: Path | str |
     if not db_path.exists():
         raise RuntimeError("Failed to convert index")
 
-    postprocess_index(db_path)
+    postprocess_index(db_path, exclude_globs=exclude_globs or ())
     prefix = project_path_prefix(document_path_prefix)
     if prefix is not None:
         prefix_document_paths(db_path, prefix)

{scip_cli-2.7.0 → scip_cli-2.8.0}/scip_cli/indexing/core.py

@@ -17,6 +17,7 @@ from ..cache import (
 )
 from ..config import load_project_config
 from ..discover import discover_golang_modules, discover_python_projects, discover_rust_crates
+from ..exclude import resolve_exclude_globs
 from .constants import DEFAULT_MAX_HEAP_MB
 from .languages import index_golang_module, index_python_project, index_rust_crate
 from .orchestrate import index_discovered_projects
@@ -87,10 +88,13 @@ def index_project(root, lang, cache_dir, *, replace=False, log=True):
     cache_dir.mkdir(parents=True, exist_ok=True)
     root = Path(root).resolve()
     env = indexer_env(root)
+    exclude_globs = resolve_exclude_globs(root)
 
     if lang == Language.TYPESCRIPT:
         projects = typescript_projects(root)
-        output_db, _indexed, skipped, total = index_typescript(root, cache_dir, projects, env, replace=replace)
+        output_db, _indexed, skipped, total = index_typescript(
+            root, cache_dir, projects, env, replace=replace, exclude_globs=exclude_globs
+        )
         if log:
             log_index_complete(
                 output_db,
@@ -110,6 +114,7 @@ def index_project(root, lang, cache_dir, *, replace=False, log=True):
             replace=replace,
             progress_noun="Python packages",
             index_one=index_python_project,
+            exclude_globs=exclude_globs,
         )
         if log:
             log_index_complete(
@@ -130,6 +135,7 @@ def index_project(root, lang, cache_dir, *, replace=False, log=True):
             replace=replace,
             progress_noun="Go modules",
             index_one=index_golang_module,
+            exclude_globs=exclude_globs,
         )
         if log:
             log_index_complete(
@@ -150,6 +156,7 @@ def index_project(root, lang, cache_dir, *, replace=False, log=True):
             replace=replace,
             progress_noun="Rust crates",
             index_one=index_rust_crate,
+            exclude_globs=exclude_globs,
         )
         if log:
             log_index_complete(

{scip_cli-2.7.0 → scip_cli-2.8.0}/scip_cli/indexing/languages.py

@@ -13,7 +13,7 @@ def _project_cwd(root: Path, project: Path) -> Path:
     return root if project == Path(".") else root / project
 
 
-def index_python_project(root, project, work_dir, env, *, output_db=None):
+def index_python_project(root, project, work_dir, env, *, output_db=None, exclude_globs=()):
     """Index one Python package directory into work_dir/index.db (or output_db when set)."""
     work_dir = Path(work_dir)
     work_dir.mkdir(parents=True, exist_ok=True)
@@ -31,13 +31,13 @@ def index_python_project(root, project, work_dir, env, *, output_db=None):
     if result.returncode != 0:
         return label, None, result.stderr.strip() or "indexing failed"
     try:
-        convert_scip_to_db(part_scip, db_path, document_path_prefix=project)
+        convert_scip_to_db(part_scip, db_path, document_path_prefix=project, exclude_globs=exclude_globs)
     finally:
         part_scip.unlink(missing_ok=True)
     return label, db_path, None
 
 
-def index_golang_module(root, module, work_dir, env, *, output_db=None):
+def index_golang_module(root, module, work_dir, env, *, output_db=None, exclude_globs=()):
     """Index one Go module directory into work_dir/index.db (or output_db when set)."""
     work_dir = Path(work_dir)
     work_dir.mkdir(parents=True, exist_ok=True)
@@ -55,13 +55,13 @@ def index_golang_module(root, module, work_dir, env, *, output_db=None):
     if result.returncode != 0:
         return label, None, result.stderr.strip() or "indexing failed"
     try:
-        convert_scip_to_db(part_scip, db_path, document_path_prefix=module)
+        convert_scip_to_db(part_scip, db_path, document_path_prefix=module, exclude_globs=exclude_globs)
     finally:
         part_scip.unlink(missing_ok=True)
     return label, db_path, None
 
 
-def index_rust_crate(root, crate, work_dir, env, *, output_db=None):
+def index_rust_crate(root, crate, work_dir, env, *, output_db=None, exclude_globs=()):
     """Index one Rust crate directory into work_dir/index.db (or output_db when set)."""
     work_dir = Path(work_dir)
     work_dir.mkdir(parents=True, exist_ok=True)
@@ -79,7 +79,7 @@ def index_rust_crate(root, crate, work_dir, env, *, output_db=None):
     if result.returncode != 0:
         return label, None, result.stderr.strip() or "indexing failed"
     try:
-        convert_scip_to_db(part_scip, db_path, document_path_prefix=crate)
+        convert_scip_to_db(part_scip, db_path, document_path_prefix=crate, exclude_globs=exclude_globs)
     finally:
         part_scip.unlink(missing_ok=True)
     return label, db_path, None

{scip_cli-2.7.0 → scip_cli-2.8.0}/scip_cli/indexing/orchestrate.py

@@ -103,6 +103,7 @@ def index_discovered_projects(
     replace: bool,
     progress_noun: str,
     index_one,
+    exclude_globs: tuple[str, ...] = (),
 ) -> tuple[Path, int, int, int]:
     """Index one SCIP unit per project path; merge when multiple part DBs."""
     output_db = index_db_path(cache_dir, replace=replace)
@@ -132,6 +133,7 @@ def index_discovered_projects(
                         project,
                         tmpdir_path / f"part-{index}",
                         env,
+                        exclude_globs=exclude_globs,
                     ): (index, project)
                     for index, project in enumerate(projects, start=1)
                 }
@@ -160,6 +162,7 @@ def index_discovered_projects(
                     cache_dir if direct_output else tmpdir_path / f"part-{index}",
                     env,
                     output_db=direct_output,
+                    exclude_globs=exclude_globs,
                 )
                 if db_path is None:
                     skipped += 1

scip_cli-2.8.0/scip_cli/indexing/postprocess.py

@@ -0,0 +1,296 @@
+"""Shrink converted SQLite indexes."""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+
+
+def _replace_table(conn, old_name: str, new_name: str) -> None:
+    conn.execute(f"DROP TABLE {old_name}")
+    conn.execute(f"ALTER TABLE {new_name} RENAME TO {old_name}")
+
+
+def _trim_unused_columns(conn, keep_external=False):
+    """Remove columns we never use; omit variable symbols while rebuilding global_symbols.
+
+    Args:
+        conn: SQLite connection
+        keep_external: If True, keep symbols without definitions (external libs)
+    """
+    conn.execute("""
+        CREATE TABLE documents_new (
+            id INTEGER PRIMARY KEY,
+            relative_path TEXT NOT NULL UNIQUE
+        )
+    """)
+    conn.execute("INSERT INTO documents_new (id, relative_path) SELECT id, relative_path FROM documents")
+    _replace_table(conn, "documents", "documents_new")
+
+    from ..symbols import sql_exclude_variable_symbols
+
+    exclude = sql_exclude_variable_symbols("symbol")
+
+    # Build the WHERE clause: exclude variables AND optionally require definitions
+    where_clauses = [exclude]
+    if not keep_external:
+        # Keep symbols that are either:
+        # 1. Defined in the project (have defn_enclosing_ranges entries)
+        # 2. Structural symbols needed for type analysis (type literals, parameters)
+        # 3. Functions/methods (even without defs, they may be project code)
+        #
+        # SCIP symbol format patterns (language-agnostic):
+        # - %typeLiteral% : Type/interface fields (e.g., Options#typeLiteral0:verbose.)
+        # - %).(% : Function parameters (e.g., greet().(name))
+        # - %().% : Functions and methods (e.g., Widget.run()., greet().)
+        where_clauses.append(
+            "EXISTS (SELECT 1 FROM defn_enclosing_ranges der WHERE der.symbol_id = global_symbols.id) "
+            + "OR symbol LIKE '%typeLiteral%' "
+            + "OR symbol LIKE '%).(%' "
+            + "OR symbol LIKE '%().%'"
+        )
+
+    where_clause = " AND ".join(where_clauses)
+
+    conn.execute("""
+        CREATE TABLE global_symbols_new (
+            id INTEGER PRIMARY KEY,
+            symbol TEXT NOT NULL UNIQUE,
+            display_name TEXT,
+            kind INTEGER
+        )
+    """)
+    conn.execute(f"""
+        INSERT INTO global_symbols_new (id, symbol, display_name, kind)
+        SELECT id, symbol, display_name, kind FROM global_symbols
+        WHERE {where_clause}
+    """)
+    _replace_table(conn, "global_symbols", "global_symbols_new")
+
+
+def _trim_mentions_to_known_symbols(conn):
+    """Copy mentions that reference retained symbols (drops variable refs without DELETE)."""
+    if not conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='mentions' LIMIT 1").fetchone():
+        return
+    conn.execute("""
+        CREATE TABLE mentions_new (
+            chunk_id INTEGER NOT NULL,
+            symbol_id INTEGER NOT NULL,
+            role INTEGER NOT NULL,
+            PRIMARY KEY (chunk_id, symbol_id, role)
+        )
+    """)
+    conn.execute("""
+        INSERT INTO mentions_new (chunk_id, symbol_id, role)
+        SELECT m.chunk_id, m.symbol_id, m.role
+        FROM mentions m
+        JOIN global_symbols g ON g.id = m.symbol_id
+    """)
+    _replace_table(conn, "mentions", "mentions_new")
+
+
+def _trim_defn_to_known_symbols(conn):
+    """Copy defn rows for retained symbols only."""
+    if not conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='defn_enclosing_ranges' LIMIT 1"
+    ).fetchone():
+        return
+    conn.execute("""
+        CREATE TABLE defn_enclosing_ranges_new (
+            id INTEGER PRIMARY KEY,
+            document_id INTEGER NOT NULL,
+            symbol_id INTEGER NOT NULL,
+            start_line INTEGER NOT NULL,
+            start_char INTEGER NOT NULL,
+            end_line INTEGER NOT NULL,
+            end_char INTEGER NOT NULL
+        )
+    """)
+    conn.execute("""
+        INSERT INTO defn_enclosing_ranges_new (
+            id, document_id, symbol_id, start_line, start_char, end_line, end_char
+        )
+        SELECT d.id, d.document_id, d.symbol_id, d.start_line, d.start_char, d.end_line, d.end_char
+        FROM defn_enclosing_ranges d
+        JOIN global_symbols g ON g.id = d.symbol_id
+    """)
+    _replace_table(conn, "defn_enclosing_ranges", "defn_enclosing_ranges_new")
+
+
+def _prune_excluded_documents(conn: sqlite3.Connection, exclude_globs: tuple[str, ...]) -> None:
+    """Drop documents matching exclude globs and cascade to chunks, mentions, defs, symbols."""
+    from ..exclude import path_matches_any_glob
+
+    if not exclude_globs:
+        return
+
+    rows = conn.execute("SELECT id, relative_path FROM documents").fetchall()
+    exclude_ids = {row[0] for row in rows if path_matches_any_glob(row[1], exclude_globs)}
+    if not exclude_ids:
+        return
+
+    conn.execute("""
+        CREATE TABLE documents_new (
+            id INTEGER PRIMARY KEY,
+            relative_path TEXT NOT NULL UNIQUE
+        )
+    """)
+    conn.execute(
+        "INSERT INTO documents_new (id, relative_path) SELECT id, relative_path FROM documents WHERE id NOT IN "
+        + f"({','.join('?' * len(exclude_ids))})",
+        tuple(exclude_ids),
+    )
+    _replace_table(conn, "documents", "documents_new")
+
+    if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='chunks' LIMIT 1").fetchone():
+        conn.execute("""
+            CREATE TABLE chunks_new (
+                id INTEGER PRIMARY KEY,
+                document_id INTEGER NOT NULL,
+                chunk_index INTEGER NOT NULL,
+                start_line INTEGER NOT NULL,
+                end_line INTEGER NOT NULL,
+                occurrences BLOB NOT NULL
+            )
+        """)
+        conn.execute(
+            "INSERT INTO chunks_new SELECT c.* FROM chunks c JOIN documents d ON c.document_id = d.id",
+        )
+        _replace_table(conn, "chunks", "chunks_new")
+
+    if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='mentions' LIMIT 1").fetchone():
+        conn.execute("""
+            CREATE TABLE mentions_new (
+                chunk_id INTEGER NOT NULL,
+                symbol_id INTEGER NOT NULL,
+                role INTEGER NOT NULL,
+                PRIMARY KEY (chunk_id, symbol_id, role)
+            )
+        """)
+        conn.execute("""
+            INSERT INTO mentions_new (chunk_id, symbol_id, role)
+            SELECT m.chunk_id, m.symbol_id, m.role
+            FROM mentions m
+            JOIN chunks c ON c.id = m.chunk_id
+        """)
+        _replace_table(conn, "mentions", "mentions_new")
+
+    if conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='defn_enclosing_ranges' LIMIT 1"
+    ).fetchone():
+        conn.execute("""
+            CREATE TABLE defn_enclosing_ranges_new (
+                id INTEGER PRIMARY KEY,
+                document_id INTEGER NOT NULL,
+                symbol_id INTEGER NOT NULL,
+                start_line INTEGER NOT NULL,
+                start_char INTEGER NOT NULL,
+                end_line INTEGER NOT NULL,
+                end_char INTEGER NOT NULL
+            )
+        """)
+        conn.execute("""
+            INSERT INTO defn_enclosing_ranges_new (
+                id, document_id, symbol_id, start_line, start_char, end_line, end_char
+            )
+            SELECT d.id, d.document_id, d.symbol_id, d.start_line, d.start_char, d.end_line, d.end_char
+            FROM defn_enclosing_ranges d
+            JOIN documents doc ON doc.id = d.document_id
+        """)
+        _replace_table(conn, "defn_enclosing_ranges", "defn_enclosing_ranges_new")
+
+    conn.execute("""
+        CREATE TABLE global_symbols_new (
+            id INTEGER PRIMARY KEY,
+            symbol TEXT NOT NULL UNIQUE,
+            display_name TEXT,
+            kind INTEGER
+        )
+    """)
+    conn.execute("""
+        INSERT INTO global_symbols_new (id, symbol, display_name, kind)
+        SELECT g.id, g.symbol, g.display_name, g.kind
+        FROM global_symbols g
+        WHERE EXISTS (SELECT 1 FROM defn_enclosing_ranges d WHERE d.symbol_id = g.id)
+           OR EXISTS (SELECT 1 FROM mentions m WHERE m.symbol_id = g.id)
+    """)
+    _replace_table(conn, "global_symbols", "global_symbols_new")
+
+    if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='mentions' LIMIT 1").fetchone():
+        conn.execute("""
+            CREATE TABLE mentions_new (
+                chunk_id INTEGER NOT NULL,
+                symbol_id INTEGER NOT NULL,
+                role INTEGER NOT NULL,
+                PRIMARY KEY (chunk_id, symbol_id, role)
+            )
+        """)
+        conn.execute("""
+            INSERT INTO mentions_new (chunk_id, symbol_id, role)
+            SELECT m.chunk_id, m.symbol_id, m.role
+            FROM mentions m
+            JOIN global_symbols g ON g.id = m.symbol_id
+        """)
+        _replace_table(conn, "mentions", "mentions_new")
+
+    if conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='defn_enclosing_ranges' LIMIT 1"
+    ).fetchone():
+        conn.execute("""
+            CREATE TABLE defn_enclosing_ranges_new (
+                id INTEGER PRIMARY KEY,
+                document_id INTEGER NOT NULL,
+                symbol_id INTEGER NOT NULL,
+                start_line INTEGER NOT NULL,
+                start_char INTEGER NOT NULL,
+                end_line INTEGER NOT NULL,
+                end_char INTEGER NOT NULL
+            )
+        """)
+        conn.execute("""
+            INSERT INTO defn_enclosing_ranges_new (
+                id, document_id, symbol_id, start_line, start_char, end_line, end_char
+            )
+            SELECT d.id, d.document_id, d.symbol_id, d.start_line, d.start_char, d.end_line, d.end_char
+            FROM defn_enclosing_ranges d
+            JOIN global_symbols g ON g.id = d.symbol_id
+        """)
+        _replace_table(conn, "defn_enclosing_ranges", "defn_enclosing_ranges_new")
+
+
+def _recreate_postprocess_indexes(conn: sqlite3.Connection) -> None:
+    """expt-convert indexes are dropped when tables are rebuilt."""
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_global_symbols_symbol ON global_symbols(symbol)")
+    if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='mentions' LIMIT 1").fetchone():
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_mentions_symbol_id_role ON mentions(symbol_id, role)")
+
+
+def postprocess_index(db_path, keep_external=None, exclude_globs: tuple[str, ...] | None = None):
+    """Shrink index: drop unused columns and prune external symbols by default.
+
+    Args:
+        db_path: Path to the SQLite database
+        keep_external: If True, keep symbols without definitions (external libs).
+                      If None, reads from SCIP_CLI_KEEP_EXTERNAL env var.
+        exclude_globs: Repo-relative globs; matching documents are removed entirely.
+    """
+    from ..sql import configure_bulk_write_connection
+
+    # Default to False unless explicitly set via parameter or env var
+    if keep_external is None:
+        keep_external = os.environ.get("SCIP_CLI_KEEP_EXTERNAL", "0") == "1"
+
+    conn = sqlite3.connect(str(db_path))
+    try:
+        configure_bulk_write_connection(conn)
+        # Rebuild tables first; indexes are recreated once at the end (expt-convert indexes
+        # are dropped with table swaps — avoid maintaining them during bulk inserts).
+        if exclude_globs:
+            _prune_excluded_documents(conn, exclude_globs)
+        _trim_unused_columns(conn, keep_external=keep_external)
+        _trim_mentions_to_known_symbols(conn)
+        _trim_defn_to_known_symbols(conn)
+        _recreate_postprocess_indexes(conn)
+        conn.commit()
+    finally:
+        conn.close()