contextual-engine 0.1.0__py3-none-any.whl

contextual/storage/migration_v2.py

@@ -0,0 +1,289 @@
+"""Phase 2 schema migration — Migration version 2.
+
+Adds documentation-specific columns to the ``chunks`` table and
+extends the FTS5 index to include ``heading_path`` for boosted BM25.
+
+HOW TO WIRE INTO migrations.py
+================================
+Find the ``run_migrations`` function in ``contextual/storage/migrations.py``
+and add ONE call inside the version dispatch block:
+
+    # Existing (version 1 was Phase 1 baseline):
+    if current_version < 1:
+        _apply_migration_1(conn)
+        record_migration(conn, 1)
+        applied += 1
+
+    # ADD THIS BLOCK:
+    if current_version < 2:
+        from contextual.storage.migration_v2 import apply_migration_2
+        apply_migration_2(conn)
+        record_migration(conn, 2)
+        applied += 1
+
+    return applied
+
+That's it. The rest of this file is self-contained.
+
+Migration strategy
+------------------
+SQLite supports ``ALTER TABLE ... ADD COLUMN`` for regular tables since 3.1.3.
+We use it for all five new ``chunks`` columns (safe, idempotent via
+``PRAGMA table_info`` guard).
+
+FTS5 virtual tables do NOT support ALTER TABLE.  The correct approach is:
+  1. Create ``fts_chunks_new`` with the extended schema.
+  2. Populate from existing ``chunks`` data (heading_path defaults to '').
+  3. Drop ``fts_chunks``.
+  4. Rename ``fts_chunks_new`` → ``fts_chunks``.
+  5. Recreate the three content-sync triggers.
+
+SQLite's ``PRAGMA writable_schema`` is NOT used — we keep everything
+through the public DDL surface for safety.
+"""
+from __future__ import annotations
+
+import sqlite3
+from sqlite3 import Connection
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+# ---------------------------------------------------------------------------
+# Column definitions added to ``chunks``
+# ---------------------------------------------------------------------------
+
+_NEW_CHUNKS_COLUMNS: list[tuple[str, str]] = [
+    # (column_name, ALTER TABLE fragment)
+    ("file_path",       "ALTER TABLE chunks ADD COLUMN file_path       TEXT    NOT NULL DEFAULT ''"),
+    ("source_type",     "ALTER TABLE chunks ADD COLUMN source_type     TEXT    NOT NULL DEFAULT 'code'"),
+    ("heading_path",    "ALTER TABLE chunks ADD COLUMN heading_path    TEXT    NOT NULL DEFAULT ''"),
+    ("heading_level",   "ALTER TABLE chunks ADD COLUMN heading_level   INTEGER NOT NULL DEFAULT 0"),
+    ("chunk_type",      "ALTER TABLE chunks ADD COLUMN chunk_type      TEXT    NOT NULL DEFAULT 'code'"),
+    ("frontmatter_json","ALTER TABLE chunks ADD COLUMN frontmatter_json TEXT   NOT NULL DEFAULT '{}'"),
+]
+
+# ---------------------------------------------------------------------------
+# New FTS5 schema (extends Phase 1 schema with heading_path and symbol_name)
+# ---------------------------------------------------------------------------
+
+_FTS5_CREATE = """
+    CREATE VIRTUAL TABLE IF NOT EXISTS fts_chunks_new USING fts5(
+        chunk_id UNINDEXED,
+        content,
+        file_path UNINDEXED,
+        symbol_name,
+        heading_path,
+        content='chunks',
+        content_rowid='id',
+        tokenize='unicode61 remove_diacritics 2 tokenchars "_-."'
+    )
+"""
+
+# Triggers keep fts_chunks_new in sync with chunks (external content table)
+_FTS5_TRIGGERS = [
+    # INSERT trigger
+    """
+    CREATE TRIGGER IF NOT EXISTS fts_chunks_ai AFTER INSERT ON chunks BEGIN
+        INSERT INTO fts_chunks(rowid, chunk_id, content, file_path, symbol_name, heading_path)
+        VALUES (new.id, new.id, new.content, '', 
+                COALESCE(new.symbol_name, ''), COALESCE(new.heading_path, ''));
+    END
+    """,
+    # DELETE trigger
+    """
+    CREATE TRIGGER IF NOT EXISTS fts_chunks_ad AFTER DELETE ON chunks BEGIN
+        INSERT INTO fts_chunks(fts_chunks, rowid, chunk_id, content, file_path, symbol_name, heading_path)
+        VALUES ('delete', old.id, old.id, old.content, '', 
+                COALESCE(old.symbol_name, ''), COALESCE(old.heading_path, ''));
+    END
+    """,
+    # UPDATE trigger
+    """
+    CREATE TRIGGER IF NOT EXISTS fts_chunks_au AFTER UPDATE ON chunks BEGIN
+        INSERT INTO fts_chunks(fts_chunks, rowid, chunk_id, content, file_path, symbol_name, heading_path)
+        VALUES ('delete', old.id, old.id, old.content, '', 
+                COALESCE(old.symbol_name, ''), COALESCE(old.heading_path, ''));
+        INSERT INTO fts_chunks(rowid, chunk_id, content, file_path, symbol_name, heading_path)
+        VALUES (new.id, new.id, new.content, '', 
+                COALESCE(new.symbol_name, ''), COALESCE(new.heading_path, ''));
+    END
+    """,
+]
+
+
+# ---------------------------------------------------------------------------
+# Public migration function
+# ---------------------------------------------------------------------------
+
+def apply_migration_2(conn: Connection) -> None:
+    """Apply Phase 2 schema migration to an existing Contextual database.
+
+    Operations (all idempotent — safe to call on an already-migrated DB):
+      1. Add five new columns to ``chunks`` (ALTER TABLE ADD COLUMN, no-op if exists).
+      2. Rebuild ``fts_chunks`` virtual table to include ``heading_path``.
+      3. Backfill the new FTS table from existing code chunks.
+      4. Drop Phase 1 triggers; install Phase 2 three-trigger set.
+      5. Add missing performance indexes.
+
+    Args:
+        conn: Open SQLite connection with write access.
+              Must have sqlite_vec and FTS5 available.
+
+    Raises:
+        sqlite3.Error: On any unrecoverable DDL failure.
+    """
+    logger.info("Applying Phase 2 migration (version 2)")
+
+    # ------------------------------------------------------------------
+    # 1. Add new columns to chunks (idempotent)
+    # ------------------------------------------------------------------
+    existing_cols = {
+        row[1]
+        for row in conn.execute("PRAGMA table_info(chunks)").fetchall()
+    }
+
+    added: list[str] = []
+    for col_name, alter_sql in _NEW_CHUNKS_COLUMNS:
+        if col_name not in existing_cols:
+            conn.execute(alter_sql)
+            added.append(col_name)
+            logger.debug("Added column to chunks", column=col_name)
+
+    if added:
+        logger.info("chunks table columns added", columns=added)
+    else:
+        logger.info("chunks table columns already present — skipping")
+
+    # Backfill file_path from the files table for existing rows
+    conn.execute("""
+        UPDATE chunks
+        SET file_path = (
+            SELECT f.path FROM files f WHERE f.id = chunks.file_id
+        )
+        WHERE file_path = '' AND file_id IS NOT NULL
+    """)
+    logger.info("Backfilled file_path on chunks from files table")
+
+    # ------------------------------------------------------------------
+    # 2. Rebuild fts_chunks to include heading_path
+    #    Skip if already rebuilt (detect by checking column count via
+    #    fts5 info query — safest approach without touching writable_schema).
+    # ------------------------------------------------------------------
+    needs_fts_rebuild = _fts_needs_rebuild(conn)
+
+    if needs_fts_rebuild:
+        logger.info("Rebuilding fts_chunks with heading_path and symbol_name columns")
+        _rebuild_fts(conn)
+    else:
+        logger.info("fts_chunks already has heading_path — skipping rebuild")
+
+    # ------------------------------------------------------------------
+    # 3. Add performance indexes (I2)
+    # ------------------------------------------------------------------
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS idx_chunks_source_type
+        ON chunks(source_type)
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS idx_chunks_heading_path
+        ON chunks(heading_path)
+        WHERE heading_path != ''
+    """)
+    conn.execute("""
+        CREATE INDEX IF NOT EXISTS idx_chunks_file_path_source
+        ON chunks(file_id, source_type)
+    """)
+
+    conn.commit()
+    logger.info("Phase 2 migration complete")
+
+
+# ---------------------------------------------------------------------------
+# FTS rebuild helpers
+# ---------------------------------------------------------------------------
+
+def _fts_needs_rebuild(conn: Connection) -> bool:
+    """Return True if fts_chunks does NOT have a heading_path column.
+
+    We probe by attempting a query that references heading_path.
+    If it raises OperationalError, the column doesn't exist yet.
+    """
+    try:
+        conn.execute(
+            "SELECT heading_path FROM fts_chunks WHERE fts_chunks MATCH 'probe_migration_v2' LIMIT 1"
+        )
+        return False
+    except sqlite3.OperationalError:
+        return True
+
+
+def _rebuild_fts(conn: Connection) -> None:
+    """Rebuild fts_chunks virtual table with heading_path support.
+
+    Steps:
+      1. Create fts_chunks_new (extended schema).
+      2. Populate from chunks table (existing rows, heading_path='').
+      3. Drop fts_chunks (and its auto-generated shadow tables).
+      4. Rename fts_chunks_new → fts_chunks.
+      5. Drop Phase 1 triggers (if they exist).
+      6. Install Phase 2 three-trigger set.
+    """
+    # Step 1: Create new FTS table
+    conn.execute(_FTS5_CREATE)
+
+    # Step 2: Backfill from existing chunks
+    # Existing code chunks have heading_path='' (DEFAULT); docs chunks
+    # will be re-indexed by DocsPipeline on first run.
+    conn.execute("""
+        INSERT INTO fts_chunks_new(rowid, chunk_id, content, file_path, symbol_name, heading_path)
+        SELECT id, id, content, '', COALESCE(symbol_name, ''), ''
+        FROM chunks
+        WHERE id NOT IN (SELECT rowid FROM fts_chunks_new)
+    """)
+    logger.debug("Backfilled fts_chunks_new from chunks table")
+
+    # Step 3: Drop old fts_chunks
+    # FTS5 drop cascades to shadow tables (_data, _idx, _content, _docsize, _config).
+    conn.execute("DROP TABLE IF EXISTS fts_chunks")
+
+    # Step 4: Rename — SQLite supports ALTER TABLE ... RENAME TO for virtual tables
+    conn.execute("ALTER TABLE fts_chunks_new RENAME TO fts_chunks")
+    logger.debug("Renamed fts_chunks_new → fts_chunks")
+
+    # Step 5: Drop Phase 1 triggers (they reference old schema)
+    for trigger in ("chunks_ai", "chunks_ad", "chunks_au",
+                    "fts_chunks_ai", "fts_chunks_ad", "fts_chunks_au"):
+        conn.execute(f"DROP TRIGGER IF EXISTS {trigger}")
+
+    # Step 6: Install Phase 2 triggers
+    for trigger_sql in _FTS5_TRIGGERS:
+        conn.execute(trigger_sql)
+
+    logger.info("fts_chunks rebuilt with heading_path column, triggers reinstalled")
+
+
+# ---------------------------------------------------------------------------
+# Standalone CLI helper (python -m contextual.storage.migration_v2)
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import sys
+    from pathlib import Path
+
+    if len(sys.argv) < 2:
+        print("Usage: python -m contextual.storage.migration_v2 <path/to/contextual.db>")
+        sys.exit(1)
+
+    db_path = Path(sys.argv[1])
+    if not db_path.exists():
+        print(f"Error: {db_path} not found")
+        sys.exit(1)
+
+    import sqlite3 as _sqlite3
+    conn = _sqlite3.connect(str(db_path))
+    conn.execute("PRAGMA journal_mode=WAL")
+    apply_migration_2(conn)
+    conn.close()
+    print(f"✓ Migration v2 applied to {db_path}")

contextual/storage/migrations.py

@@ -0,0 +1,316 @@
+"""SQLite migration runner for schema version management.
+
+Tracks schema versions and applies pending migrations in order.
+Migrations:
+- v1: Core bi-temporal schema (entities, facts, episodes)
+- v2: Phase 1 codebase indexing (code_symbols, indexing_state, vec0, FTS5)
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import time
+from typing import TYPE_CHECKING
+
+from contextual.core.errors import ErrorCode, StorageError, storage_context
+from contextual.observability.logging import get_logger
+
+
+if TYPE_CHECKING:
+    from sqlite3 import Connection
+
+logger = get_logger(__name__)
+
+
+def get_current_version(conn: Connection) -> int:
+    """Get current schema version.
+
+    Reads from schema_version table. Returns 0 if table doesn't exist
+    or is empty (indicating fresh database).
+
+    Args:
+        conn: SQLite connection.
+
+    Returns:
+        Current schema version (0 if no version recorded).
+
+    Raises:
+        StorageError: If version query fails.
+    """
+    try:
+        cursor = conn.cursor()
+
+        # Check if schema_version table exists.
+        cursor.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name='schema_version'",
+        )
+        if not cursor.fetchone():
+            return 0
+
+        # Get the latest recorded version.
+        cursor.execute("SELECT MAX(version) FROM schema_version")
+        result = cursor.fetchone()
+
+        if result is None or result[0] is None:
+            return 0
+
+        return int(result[0])
+
+    except sqlite3.Error as e:
+        msg = f"Failed to get schema version: {e}"
+        raise StorageError(
+            code=ErrorCode.STORAGE_MIGRATION_FAILED,
+            message=msg,
+            context=storage_context(query="get_current_version"),
+        ) from e
+
+
+def record_migration(conn: Connection, version: int) -> None:
+    """Record that a migration was applied.
+
+    Inserts version and timestamp into schema_version table.
+
+    Args:
+        conn: SQLite connection.
+        version: Migration version number.
+
+    Raises:
+        StorageError: If recording fails.
+    """
+    try:
+        cursor = conn.cursor()
+        now_ms = int(time.time() * 1000)
+
+        cursor.execute(
+            "INSERT INTO schema_version (version, applied_at) VALUES (?, ?)",
+            (version, now_ms),
+        )
+        conn.commit()
+
+    except sqlite3.Error as e:
+        msg = f"Failed to record migration version {version}: {e}"
+        raise StorageError(
+            code=ErrorCode.STORAGE_MIGRATION_FAILED,
+            message=msg,
+            context=storage_context(query=f"record_migration({version})"),
+        ) from e
+
+
+def apply_initial_schema(conn: Connection) -> None:
+    """Apply initial schema to a fresh database.
+
+    Creates all tables, indexes, and FTS5 structures.
+    Records this as version 1.
+
+    Args:
+        conn: SQLite connection.
+
+    Raises:
+        StorageError: If schema application fails.
+    """
+    from contextual.storage.schema import apply_schema
+
+    try:
+        # Apply complete schema for version 1.
+        apply_schema(conn)
+
+        # Record the schema as version 1.
+        record_migration(conn, 1)
+
+    except StorageError:
+        # Re-raise storage errors as-is.
+        raise
+    except Exception as e:
+        msg = f"Failed to apply initial schema: {e}"
+        raise StorageError(
+            code=ErrorCode.STORAGE_MIGRATION_FAILED,
+            message=msg,
+            context=storage_context(),
+        ) from e
+
+
+def apply_migration_v2(conn: Connection) -> None:
+    """Apply migration v2: Phase 1 codebase indexing tables.
+
+    Adds:
+    - code_symbols table (function/class/method metadata)
+    - indexing_state table (tracks repo indexing status)
+    - chunks table (code chunks with embeddings)
+    - files table (indexed files)
+    - vec0 virtual table (768-dim vectors for jina-v2-code)
+    - fts_chunks virtual table (FTS5 full-text search)
+
+    Args:
+        conn: SQLite connection.
+
+    Raises:
+        StorageError: If migration fails.
+    """
+    try:
+        cursor = conn.cursor()
+
+        # FILES TABLE - tracks indexed files
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS files (
+                id INTEGER PRIMARY KEY,
+                path TEXT NOT NULL UNIQUE,
+                language TEXT NOT NULL,
+                mtime INTEGER NOT NULL,
+                content_hash TEXT NOT NULL,
+                indexed_at INTEGER NOT NULL
+            ) STRICT
+        """)
+
+        # CHUNKS TABLE - code chunks with metadata
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS chunks (
+                id INTEGER PRIMARY KEY,
+                file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+                start_line INTEGER NOT NULL,
+                end_line INTEGER NOT NULL,
+                symbol_name TEXT,
+                symbol_type TEXT,
+                content TEXT NOT NULL,
+                content_hash TEXT NOT NULL UNIQUE,
+                embedding_id INTEGER,
+                indexed_at INTEGER NOT NULL
+            ) STRICT
+        """)
+
+        # CODE_SYMBOLS TABLE - extracted symbols with metadata
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS code_symbols (
+                id INTEGER PRIMARY KEY,
+                chunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
+                name TEXT NOT NULL,
+                kind TEXT NOT NULL CHECK (
+                    kind IN ('function','class','method','interface','struct',
+                             'enum','type_alias','variable','import')
+                ),
+                signature TEXT,
+                docstring TEXT,
+                UNIQUE(chunk_id, name, kind)
+            ) STRICT
+        """)
+
+        # INDEXING_STATE TABLE - repo-level indexing metadata
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS indexing_state (
+                repo_root TEXT PRIMARY KEY,
+                last_full_index_at INTEGER NOT NULL,
+                last_incremental_at INTEGER,
+                total_files INTEGER NOT NULL,
+                total_chunks INTEGER NOT NULL,
+                total_symbols INTEGER NOT NULL
+            ) STRICT
+        """)
+
+        # Indexes for performance
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_id)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_symbols_name ON code_symbols(name)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_symbols_kind ON code_symbols(kind)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_symbols_chunk ON code_symbols(chunk_id)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_files_path ON files(path)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_files_content_hash ON files(content_hash)")
+
+        # VEC0 VIRTUAL TABLE - 768-dim vectors (jina-v2-code)
+        # Note: vec0 extension must be loaded before this runs
+        try:
+            cursor.execute("""
+                CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks
+                USING vec0(
+                    chunk_id INTEGER PRIMARY KEY,
+                    embedding FLOAT[768]
+                )
+            """)
+        except sqlite3.OperationalError as e:
+            # vec0 not available - log warning but don't fail migration
+            logger.warning(
+                "vec0 extension not available - vector search disabled",
+                error=str(e)
+            )
+
+        # FTS5 VIRTUAL TABLE - full-text search on chunks
+        # Using manual sync approach instead of triggers to avoid FTS5 transaction issues
+        cursor.execute("""
+            CREATE VIRTUAL TABLE IF NOT EXISTS fts_chunks USING fts5(
+                chunk_id UNINDEXED,
+                content,
+                symbol_name,
+                tokenize='unicode61 remove_diacritics 2 tokenchars "_"'
+            )
+        """)
+
+        # EMBEDDING CACHE TABLE
+        from contextual.embedding.cache import initialize_cache
+        try:
+            initialize_cache(conn)
+            logger.info("Embedding cache initialized")
+        except Exception as e:
+            logger.warning("Failed to initialize embedding cache", error=str(e))
+        
+
+        # NOTE: Triggers disabled to avoid "unsafe use of virtual table" errors with FTS5
+        # FTS5 sync is now handled manually in index_writer.py after chunk insertion
+
+        conn.commit()
+
+        logger.info("Migration v2 applied successfully")
+
+    except sqlite3.Error as e:
+        conn.rollback()
+        msg = f"Failed to apply migration v2: {e}"
+        raise StorageError(
+            code=ErrorCode.STORAGE_MIGRATION_FAILED,
+            message=msg,
+            context=storage_context(query="apply_migration_v2"),
+        ) from e
+
+
+def run_migrations(conn: Connection) -> int:
+    """Run any pending migrations.
+
+    Checks current version and applies necessary migrations.
+
+    Args:
+        conn: SQLite connection.
+
+    Returns:
+        Number of migrations applied.
+
+    Raises:
+        StorageError: If migrations fail.
+    """
+    current_version = get_current_version(conn)
+    migrations_applied = 0
+
+    # Version 0 -> 1: initial schema.
+    if current_version == 0:
+        apply_initial_schema(conn)
+        migrations_applied += 1
+        current_version = 1
+
+    # Version 1 -> 2: Phase 1 codebase indexing
+    if current_version == 1:
+        apply_migration_v2(conn)
+        record_migration(conn, 2)
+        migrations_applied += 1
+        current_version = 2
+
+    # Version 2 -> 3: Phase 2 docs schema (heading_path columns, FTS rebuild)
+    if current_version == 2:
+        from contextual.storage.migration_v2 import apply_migration_2
+        apply_migration_2(conn)
+        record_migration(conn, 3)
+        migrations_applied += 1
+        current_version = 3
+
+    if migrations_applied > 0:
+        logger.info(
+            "Migrations complete",
+            applied=migrations_applied,
+            current_version=current_version
+        )
+
+    return migrations_applied