sweet-search 2.5.2 → 2.5.4

package/core/indexing/indexer-ann.js

@@ -7,12 +7,14 @@ import { existsSync, openSync, fsyncSync, closeSync, writeFileSync, readFileSync
 import path from 'path';
 
 import { DB_PATHS, HNSW_CONFIG, BINARY_HNSW_CONFIG } from '../infrastructure/config/index.js';
+import { chunkedIn } from '../infrastructure/db-utils.js';
 import { HNSWIndex } from '../vector-store/hnsw-index.js';
 import { LateInteractionIndex } from '../ranking/late-interaction-index.js';
 import { truncateForHNSW, getEmbeddings, getModelInfo, fisherYatesShuffle } from '../embedding/embedding-service.js';
 import { buildFromCodebaseDb as buildQuantizedArtifacts, shouldSkipArtifactRebuild, updateArtifactState, ARTIFACT_THRESHOLDS } from './artifact-builder.js';
 import { log, logProgress } from './indexer-utils.js';
 import { JAVA_FAMILY } from './ast-chunker.js';
+import { isIndexAcceleratorAvailable } from './model-pool.js';
 
 // =============================================================================
 // DURABLE WRITE HELPERS (Phase E — fsync ordering for checkpoint safety)
@@ -60,6 +62,28 @@ export function pickLiInput(chunk) {
   return chunk.li_greedy_text || chunk.embedding_text || chunk.li_text || chunk.text || chunk.content || '';
 }
 
+function chunkFilePath(chunk) {
+  return firstSafeRelativePath(
+    chunk?.metadata?.relative_path,
+    chunk?.metadata?.path,
+    chunk?.metadata?.file_path,
+    chunk?.file,
+    chunk?.metadata?.file,
+  ) || '';
+}
+
+function firstSafeRelativePath(...candidates) {
+  for (const candidate of candidates) {
+    if (typeof candidate !== 'string') continue;
+    const normalized = candidate.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/');
+    if (!normalized || normalized === '.' || normalized.startsWith('/')) continue;
+    if (/^[A-Za-z]:\//.test(normalized)) continue;
+    if (normalized === '..' || normalized.startsWith('../') || normalized.includes('/../')) continue;
+    return normalized;
+  }
+  return null;
+}
+
 function fsyncFile(filePath) {
   const fd = openSync(filePath, 'r');
   try { fsyncSync(fd); } finally { closeSync(fd); }
@@ -108,13 +132,38 @@ function cleanupCheckpoint(indexPath) {
 // list at the exemplar's rank position.
 const ALIAS_FILTER_SQL = "json_extract(metadata, '$.exemplarId') IS NULL";
 
+function hasVectorColumn(db, column) {
+  try {
+    return db.prepare('PRAGMA table_info(vectors)').all().some((col) => col.name === column);
+  } catch (_err) {
+    return false;
+  }
+}
+
+function aliasFilterSql(alias = '') {
+  if (!alias) return ALIAS_FILTER_SQL;
+  const prefix = alias ? `${alias}.` : '';
+  return `json_extract(${prefix}metadata, '$.exemplarId') IS NULL`;
+}
+
+function liveVectorSql(db, alias = '') {
+  if (!hasVectorColumn(db, 'epoch_retired')) return '1=1';
+  const prefix = alias ? `${alias}.` : '';
+  return `${prefix}epoch_retired IS NULL`;
+}
+
+function vectorIndexWhere(db, alias = '') {
+  return `${aliasFilterSql(alias)} AND ${liveVectorSql(db, alias)}`;
+}
+
 function* streamVectorsFromDb(db, _dim, order = 'sequential') {
+  const vectorWhere = vectorIndexWhere(db);
   if (order !== 'sequential') {
     db.exec('CREATE TEMP TABLE IF NOT EXISTS hnsw_order (pos INTEGER PRIMARY KEY, vector_rowid INTEGER)');
     db.exec('DELETE FROM hnsw_order');
 
     const rowidRows = db
-      .prepare(`SELECT rowid FROM vectors WHERE ${ALIAS_FILTER_SQL} ORDER BY rowid`)
+      .prepare(`SELECT rowid FROM vectors WHERE ${vectorWhere} ORDER BY rowid`)
       .all();
     let indices = rowidRows.map((r) => r.rowid);
 
@@ -122,7 +171,7 @@ function* streamVectorsFromDb(db, _dim, order = 'sequential') {
       fisherYatesShuffle(indices);
     } else if (order === 'diversity') {
       const pathRows = db
-        .prepare(`SELECT rowid, file_path FROM vectors WHERE ${ALIAS_FILTER_SQL} ORDER BY rowid`)
+        .prepare(`SELECT rowid, file_path FROM vectors WHERE ${vectorWhere} ORDER BY rowid`)
         .all();
       const filePaths = pathRows.map((r) => r.file_path);
       const permutationPositions = diversityFirstPermutationRowids(filePaths);
@@ -155,7 +204,7 @@ function* streamVectorsFromDb(db, _dim, order = 'sequential') {
     db.exec('DROP TABLE IF EXISTS temp.hnsw_order');
   } else {
     const stmt = db.prepare(
-      `SELECT rowid, id, file_path, embedding, metadata FROM vectors WHERE ${ALIAS_FILTER_SQL} ORDER BY rowid`,
+      `SELECT rowid, id, file_path, embedding, metadata FROM vectors WHERE ${vectorWhere} ORDER BY rowid`,
     );
     for (const row of stmt.iterate()) {
       yield {
@@ -192,12 +241,16 @@ function* streamVectorsFromDb(db, _dim, order = 'sequential') {
 export function decideHybridDispatcher({
   env = process.env,
   parallelLateInteraction = false,
+  acceleratorAvailable = true,
 } = {}) {
   const hybridEnv = (env.SWEET_SEARCH_LI_HYBRID ?? '').trim().toLowerCase();
   const hybridEnabled = hybridEnv === '1' || hybridEnv === 'true' || hybridEnv === 'on';
   if (!hybridEnabled) {
     return { armed: false, reason: 'not-enabled' };
   }
+  if (!acceleratorAvailable) {
+    return { armed: false, reason: 'no-accelerator' };
+  }
   // SWEET_SEARCH_LI_USE_CPU implies single-encoder CPU path — skip the
   // bidirectional cursor (which would still try to use the GPU encoder).
   if (env.SWEET_SEARCH_LI_USE_CPU === '1') {
@@ -395,14 +448,29 @@ export async function incrementalUpdateHNSW(dbPath, changedFiles, dryRun = false
   const Database = (await import('better-sqlite3')).default;
   const db = new Database(dbPath, { readonly: true });
 
-  const changedFileSet = new Set(changedFiles || []);
-  const placeholders = [...changedFileSet].map(() => '?').join(',');
-  const stmt = changedFileSet.size > 0
-    ? db.prepare(`SELECT rowid, id, file_path, embedding, metadata FROM vectors WHERE ${ALIAS_FILTER_SQL} AND file_path IN (${placeholders}) ORDER BY rowid`)
-    : db.prepare(`SELECT rowid, id, file_path, embedding, metadata FROM vectors WHERE ${ALIAS_FILTER_SQL} ORDER BY rowid`);
-
-  const rows = changedFileSet.size > 0 ? stmt.all(...changedFileSet) : [];
-  const totalNew = changedFileSet.size > 0 ? rows.length : 0;
+  const changedFileList = [...new Set(changedFiles || [])];
+  // Chunk the IN(?,?,...) clause to stay under SQLite's bound-parameter
+  // limit (default 32766, historic floor 999). Without chunking, a single
+  // indexing pass over >~32k changed files crashes with "too many SQL
+  // variables" — observed in production on CoSQA+ (51k docs) and BRIGHT
+  // (528k docs). See core/infrastructure/db-utils.js for the helper.
+  let rows = [];
+  if (changedFileList.length > 0) {
+    rows = chunkedIn(
+      db,
+      `SELECT rowid, id, file_path, embedding, metadata
+         FROM vectors
+        WHERE ${vectorIndexWhere(db)}
+          AND file_path IN (__IN_PLACEHOLDERS__)
+        ORDER BY rowid`,
+      changedFileList,
+    );
+    // Each batch is ORDER BY rowid internally, but batch boundaries break
+    // global monotonicity. The HNSW insertion loop below relies on rowid
+    // order for deterministic graph construction — re-sort explicitly.
+    rows.sort((a, b) => a.rowid - b.rowid);
+  }
+  const totalNew = rows.length;
 
   log(`Adding ${totalNew} new entries...`, 'yellow');
   let added = 0;
@@ -455,7 +523,7 @@ export async function buildHNSWIndex(dbPath, dryRun = false) {
   const db = new Database(dbPath, orderMode === 'sequential' ? { readonly: true } : {});
 
   const totalVectors = db
-    .prepare(`SELECT COUNT(*) as c FROM vectors WHERE ${ALIAS_FILTER_SQL}`)
+    .prepare(`SELECT COUNT(*) as c FROM vectors WHERE ${vectorIndexWhere(db)}`)
     .get().c;
   if (totalVectors === 0) {
     db.close();
@@ -499,7 +567,10 @@ export async function buildHNSWIndex(dbPath, dryRun = false) {
         // vectors already in the checkpoint. Without this, add() reuses keys
         // from 0 and the final .meta.json would be incomplete.
         const metaStmt = db.prepare(
-          'SELECT id, file_path, metadata FROM vectors WHERE rowid <= ? ORDER BY rowid'
+          `SELECT id, file_path, metadata
+             FROM vectors
+            WHERE rowid <= ? AND ${vectorIndexWhere(db)}
+            ORDER BY rowid`
         );
         let restoredKey = 0;
         for (const row of metaStmt.iterate(resumeFromRowId)) {
@@ -592,6 +663,7 @@ export async function buildHNSWIndex(dbPath, dryRun = false) {
     }
 
     await index.save();
+    await index.clearStaleBitmap();
     buildCompleted = true;
 
     // Clean up checkpoint files after successful completion
@@ -830,6 +902,7 @@ export async function buildLateInteractionIndex(chunks, dryRun = false, filesToR
     const hybridDecision = decideHybridDispatcher({
       env: process.env,
       parallelLateInteraction: EMBEDDING_CONFIG.parallelLateInteraction === true,
+      acceleratorAvailable: isIndexAcceleratorAvailable(),
     });
     if (!hybridDecision.armed && hybridDecision.reason === 'metal-contended-by-embed') {
       log(
@@ -837,6 +910,11 @@ export async function buildLateInteractionIndex(chunks, dryRun = false, filesToR
         + 'OR SWEET_SEARCH_EMBED_USE_CPU=1 (Metal queue is shared with parallel embed phase)',
         'yellow'
       );
+    } else if (!hybridDecision.armed && hybridDecision.reason === 'no-accelerator') {
+      log(
+        'LateInteraction hybrid: ignored — no inference accelerator detected; using ORT CPU',
+        'yellow'
+      );
     }
     const hybridDisabled = !hybridDecision.armed;
 
@@ -913,7 +991,7 @@ export async function buildLateInteractionIndex(chunks, dryRun = false, filesToR
         const tokens = tokenArrays[j];
         if (tokens && tokens.length > 0) {
           await liIndex.add(chunk.id, tokens, {
-            file: chunk.file,
+            file: chunkFilePath(chunk),
             name: chunk.metadata?.symbol,
             type: chunk.metadata?.chunk_type,
             startLine: chunk.metadata?.line_start || null,
@@ -1018,7 +1096,7 @@ export async function buildLateInteractionIndex(chunks, dryRun = false, filesToR
         continue;
       }
       liIndex.addAlias(alias.id, exemplarId, clusterId, {
-        file: alias.file,
+        file: chunkFilePath(alias),
         name: alias.metadata?.symbol,
         type: alias.metadata?.chunk_type,
         startLine: alias.metadata?.line_start || null,
@@ -1042,6 +1120,12 @@ export async function buildLateInteractionIndex(chunks, dryRun = false, filesToR
   return { ...liStats, added: totalAdded, removed, saveToPath };
 }
 
+export const __TEST__ = {
+  chunkFilePath,
+  vectorIndexWhere,
+  liveVectorSql,
+};
+
 // =============================================================================
 // PHASE 5: BINARY HNSW + INT8 QUANTIZED ARTIFACTS
 // =============================================================================

package/core/indexing/indexer-build.js

@@ -11,8 +11,12 @@ import path from 'path';
 import { DB_PATHS, EMBEDDING_CONFIG, PROJECT_ROOT } from '../infrastructure/config/index.js';
 import { GraphExtractor, createGraphSchema, insertGraph } from '../graph/graph-extractor.js';
 import { resolveRelationshipTargets } from '../graph/relationship-resolver.js';
+import { populatePageRankColumn } from '../graph/structural-pagerank.js';
 import { getEmbeddings, getModelInfo } from '../embedding/embedding-service.js';
 import { configureJournalMode, checkpointWal, atomicSwapDatabase, log, logProgress } from './indexer-utils.js';
+import { assignStructuralIds } from '../incremental-indexing/domain/chunk-identity.mjs';
+import { chunkInputHashes } from '../incremental-indexing/domain/encoder-input.mjs';
+import { migrateVectorsSchema } from '../incremental-indexing/infrastructure/schema-migrations.mjs';
 
 // =============================================================================
 // CHUNK ENRICHMENT — scope chains + imports from code-graph.db
@@ -61,7 +65,7 @@ async function enrichChunksFromGraph(chunks, ASTChunker) {
     let enriched = 0;
 
     for (const chunk of chunks) {
-      const filePath = chunk.file || chunk.metadata?.path;
+      const filePath = chunkFilePath(chunk);
       if (!filePath) continue;
 
       // Only enrich chunks with a known symbol (skip generic 'unknown' text chunks)
@@ -187,6 +191,14 @@ export async function buildCodeGraph(files, dryRun = false) {
   log('Resolving relationship targets...', 'yellow');
   const resolutionStats = resolveRelationshipTargets(db);
 
+  log('Computing entity PageRank for structural ranking...', 'yellow');
+  try {
+    const prStats = populatePageRankColumn(db);
+    log(`✓ PageRank populated: ${prStats.written}/${prStats.entities} entities in ${prStats.ms}ms`, 'green');
+  } catch (err) {
+    log(`⚠ PageRank population failed (non-fatal): ${err.message}`, 'yellow');
+  }
+
   // Update query planner statistics before closing (SQLite 3.46+).
   // Best-effort only; failure should not strand the temp DB handle.
   closeWithOptimize(db, 'code graph build');
@@ -224,6 +236,7 @@ export function createVectorSchema(db) {
   `);
   db.exec('CREATE INDEX IF NOT EXISTS idx_vectors_session ON vectors(session_id)');
   db.exec('CREATE INDEX IF NOT EXISTS idx_vectors_file_path ON vectors(file_path)');
+  migrateVectorsSchema(db);
 }
 
 export function ensureVectorSchema(db) {
@@ -254,25 +267,30 @@ export function ensureVectorSchema(db) {
     db.exec('CREATE INDEX IF NOT EXISTS idx_vectors_file_path ON vectors(file_path)');
     log('  Schema migration complete', 'dim');
   }
+  migrateVectorsSchema(db);
 }
 
-export function buildInsertItems(chunks, embeddings, modelInfo) {
+export function buildInsertItems(chunks, embeddings, modelInfo, annotations = null, options = {}) {
   const items = [];
+  const chunkAnnotations = annotations || annotateChunksForVectorInsert(chunks);
+  const epochWritten = Number.isInteger(options.epochWritten) ? options.epochWritten : 0;
   for (let i = 0; i < chunks.length; i++) {
     const chunk = chunks[i];
     const embedding = embeddings[i];
 
     if (!embedding || embedding.length === 0) continue;
+    const ann = chunkAnnotations[i];
+    const filePath = chunkFilePath(chunk);
 
     items.push({
       id: chunk.id,
-      filePath: chunk.file,
+      filePath,
       embeddingBlob: embedding instanceof Float32Array
         ? Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength)
         : Buffer.from(new Float32Array(embedding).buffer),
       text: (chunk.text || chunk.content || '').slice(0, 2000),
       metadata: JSON.stringify({
-        file: chunk.file,
+        file: filePath,
         type: chunk.metadata?.chunk_type || 'code',
         name: chunk.metadata?.symbol || null,
         startLine: chunk.metadata?.line_start || null,
@@ -289,11 +307,117 @@ export function buildInsertItems(chunks, embeddings, modelInfo) {
       sessionId: `codebase-v22-${modelInfo.provider}`,
       tags: JSON.stringify(['codebase', chunk.metadata?.language || 'unknown']),
       createdAt: new Date().toISOString(),
+      chunkStructId: ann?.chunkStructId || '',
+      chunkTextHash: ann?.hashes?.chunk_text_hash || '',
+      embeddingInputHash: ann?.hashes?.embedding_input_hash || '',
+      liInputHash: ann?.hashes?.li_input_hash || '',
+      metadataFingerprint: ann?.hashes?.metadata_fingerprint || '',
+      logicalChunkId: ann?.chunkStructId || chunk.id,
+      epochWritten,
+      epochRetired: null,
     });
   }
   return items;
 }
 
+function chunkFilePath(chunk) {
+  return firstSafeRelativePath(
+    chunk?.metadata?.relative_path,
+    chunk?.metadata?.path,
+    chunk?.metadata?.file_path,
+    chunk?.file,
+    chunk?.metadata?.file,
+  ) || '';
+}
+
+function firstSafeRelativePath(...candidates) {
+  for (const candidate of candidates) {
+    if (typeof candidate !== 'string') continue;
+    const normalized = candidate.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/');
+    if (!normalized || normalized === '.' || normalized.startsWith('/')) continue;
+    if (/^[A-Za-z]:\//.test(normalized)) continue;
+    if (normalized === '..' || normalized.startsWith('../') || normalized.includes('/../')) continue;
+    return normalized;
+  }
+  return null;
+}
+
+function annotateChunksForVectorInsert(chunks) {
+  const annotations = new Array(chunks.length);
+  const byFile = new Map();
+  for (let i = 0; i < chunks.length; i++) {
+    const filePath = chunkFilePath(chunks[i]);
+    if (!byFile.has(filePath)) byFile.set(filePath, []);
+    byFile.get(filePath).push(i);
+  }
+  for (const [filePath, indices] of byFile.entries()) {
+    const fileChunks = indices.map((idx) => chunks[idx]);
+    const ids = assignStructuralIds(fileChunks, filePath);
+    for (let i = 0; i < indices.length; i++) {
+      const idx = indices[i];
+      annotations[idx] = {
+        ...ids[i],
+        hashes: chunkInputHashes(chunks[idx]),
+      };
+    }
+  }
+  return annotations;
+}
+
+function vectorInsertColumns(db) {
+  const columns = new Set(db.prepare('PRAGMA table_info(vectors)').all().map((c) => c.name));
+  return [
+    'id',
+    'file_path',
+    'embedding',
+    'text',
+    'metadata',
+    'session_id',
+    'tags',
+    'created_at',
+    'chunk_struct_id',
+    'chunk_text_hash',
+    'embedding_input_hash',
+    'li_input_hash',
+    'metadata_fingerprint',
+    'logical_chunk_id',
+    'epoch_written',
+    'epoch_retired',
+  ].filter((column) => columns.has(column));
+}
+
+function vectorInsertValue(item, column) {
+  switch (column) {
+    case 'id': return item.id;
+    case 'file_path': return item.filePath;
+    case 'embedding': return item.embeddingBlob;
+    case 'text': return item.text;
+    case 'metadata': return item.metadata;
+    case 'session_id': return item.sessionId;
+    case 'tags': return item.tags;
+    case 'created_at': return item.createdAt;
+    case 'chunk_struct_id': return item.chunkStructId ?? '';
+    case 'chunk_text_hash': return item.chunkTextHash ?? '';
+    case 'embedding_input_hash': return item.embeddingInputHash ?? '';
+    case 'li_input_hash': return item.liInputHash ?? '';
+    case 'metadata_fingerprint': return item.metadataFingerprint ?? '';
+    case 'logical_chunk_id': return item.logicalChunkId ?? item.chunkStructId ?? item.id;
+    case 'epoch_written': return item.epochWritten ?? 0;
+    case 'epoch_retired': return item.epochRetired ?? null;
+    default: return item[column];
+  }
+}
+
+function prepareVectorInsert(db) {
+  const columns = vectorInsertColumns(db);
+  const quoted = columns.map((column) => `"${column}"`).join(', ');
+  const placeholders = columns.map(() => '?').join(', ');
+  return {
+    columns,
+    stmt: db.prepare(`INSERT OR REPLACE INTO vectors (${quoted}) VALUES (${placeholders})`),
+  };
+}
+
 /**
  * Insert alias rows that reuse their exemplar's embedding instead of running
  * the embedding model. The exemplar must already be in the `vectors` table;
@@ -307,23 +431,11 @@ export function insertAliasVectors(db, aliases, modelInfo) {
     'SELECT embedding, metadata FROM vectors WHERE id = ?'
   );
 
-  const stmt = db.prepare(`
-    INSERT OR REPLACE INTO vectors (id, file_path, embedding, text, metadata, session_id, tags, created_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-  `);
+  const { stmt, columns } = prepareVectorInsert(db);
 
   const insertBatch = db.transaction((items) => {
     for (const item of items) {
-      stmt.run(
-        item.id,
-        item.filePath,
-        item.embeddingBlob,
-        item.text,
-        item.metadata,
-        item.sessionId,
-        item.tags,
-        item.createdAt,
-      );
+      stmt.run(...columns.map((column) => vectorInsertValue(item, column)));
     }
   });
 
@@ -344,11 +456,13 @@ export function insertAliasVectors(db, aliases, modelInfo) {
   }
 
   const items = [];
+  const annotations = annotateChunksForVectorInsert(aliases);
   const nowIso = new Date().toISOString();
   let missing = 0;
   let dimension = null;
 
-  for (const alias of aliases) {
+  for (let i = 0; i < aliases.length; i++) {
+    const alias = aliases[i];
     const exemplarId = alias.metadata?.exemplarId;
     if (!exemplarId) continue;
     const row = fetchExemplar.get(exemplarId);
@@ -359,14 +473,16 @@ export function insertAliasVectors(db, aliases, modelInfo) {
     if (dimension === null) {
       dimension = Math.floor(row.embedding.length / 4);
     }
+    const ann = annotations[i];
+    const filePath = chunkFilePath(alias);
 
     items.push({
       id: alias.id,
-      filePath: alias.file,
+      filePath,
       embeddingBlob: row.embedding, // copy exemplar's Float32 BLOB verbatim
       text: (alias.text || alias.content || '').slice(0, 2000),
       metadata: JSON.stringify({
-        file: alias.file,
+        file: filePath,
         type: alias.metadata?.chunk_type || 'code',
         name: alias.metadata?.symbol || null,
         startLine: alias.metadata?.line_start || null,
@@ -382,6 +498,14 @@ export function insertAliasVectors(db, aliases, modelInfo) {
       sessionId: `codebase-v22-${modelInfo.provider}`,
       tags: JSON.stringify(['codebase', alias.metadata?.language || 'unknown']),
       createdAt: nowIso,
+      chunkStructId: ann?.chunkStructId || '',
+      chunkTextHash: ann?.hashes?.chunk_text_hash || '',
+      embeddingInputHash: ann?.hashes?.embedding_input_hash || '',
+      liInputHash: ann?.hashes?.li_input_hash || '',
+      metadataFingerprint: ann?.hashes?.metadata_fingerprint || '',
+      logicalChunkId: ann?.chunkStructId || alias.id,
+      epochWritten: 0,
+      epochRetired: null,
     });
   }
 
@@ -397,48 +521,36 @@ export function insertAliasVectors(db, aliases, modelInfo) {
   return items.length;
 }
 
-export function insertVectors(db, chunks, embeddings, modelInfo) {
+export function insertVectorItems(db, items) {
   const BATCH_INSERT_SIZE = 2000;
 
-  const stmt = db.prepare(`
-    INSERT OR REPLACE INTO vectors (id, file_path, embedding, text, metadata, session_id, tags, created_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-  `);
+  const { stmt, columns } = prepareVectorInsert(db);
 
   const insertBatch = db.transaction((items) => {
     for (const item of items) {
-      stmt.run(
-        item.id,
-        item.filePath,
-        item.embeddingBlob,
-        item.text,
-        item.metadata,
-        item.sessionId,
-        item.tags,
-        item.createdAt
-      );
+      stmt.run(...columns.map((column) => vectorInsertValue(item, column)));
     }
   });
 
-  const items = buildInsertItems(chunks, embeddings, modelInfo);
-
   for (let i = 0; i < items.length; i += BATCH_INSERT_SIZE) {
     insertBatch(items.slice(i, i + BATCH_INSERT_SIZE));
   }
 }
 
+export function insertVectors(db, chunks, embeddings, modelInfo, annotations = null, options = {}) {
+  insertVectorItems(db, buildInsertItems(chunks, embeddings, modelInfo, annotations, options));
+}
+
 export async function pipelinedEmbedAndInsert(db, allChunks, texts, batchSize, modelInfo, logProgressFn, embeddingOptions = {}, logFn, writeFlushRows = 128) {
   let writeBuffer = [];
   let embeddingCount = 0;
+  const allAnnotations = annotateChunksForVectorInsert(allChunks);
 
-  const stmt = db.prepare(`
-    INSERT OR REPLACE INTO vectors (id, file_path, embedding, text, metadata, session_id, tags, created_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-  `);
+  const { stmt, columns } = prepareVectorInsert(db);
 
   const insertBatch = db.transaction((items) => {
     for (const item of items) {
-      stmt.run(item.id, item.filePath, item.embeddingBlob, item.text, item.metadata, item.sessionId, item.tags, item.createdAt);
+      stmt.run(...columns.map((column) => vectorInsertValue(item, column)));
     }
   });
 
@@ -458,6 +570,7 @@ export async function pipelinedEmbedAndInsert(db, allChunks, texts, batchSize, m
   for (let i = 0; i < texts.length; i += batchSize) {
     const batch = texts.slice(i, i + batchSize);
     const batchChunks = allChunks.slice(i, i + batchSize);
+    const batchAnnotations = allAnnotations.slice(i, i + batchSize);
 
     // Overlap: flush accumulated writes while embedding is in-flight
     const batchResultsPromise = getEmbeddings(batch, progressOptions);
@@ -470,7 +583,7 @@ export async function pipelinedEmbedAndInsert(db, allChunks, texts, batchSize, m
     const batchEmbeddings = batchResults.map(r => r.embedding);
     embeddingCount += batchEmbeddings.length;
 
-    const batchItems = buildInsertItems(batchChunks, batchEmbeddings, modelInfo);
+    const batchItems = buildInsertItems(batchChunks, batchEmbeddings, modelInfo, batchAnnotations);
     writeBuffer.push(...batchItems);
 
     if (!useInternalProgress) {
@@ -549,7 +662,7 @@ export async function chunkFiles(files) {
     if (chunk.embedding_text) {
       return chunk.embedding_text.slice(0, _embCap);
     }
-    return `${chunk.file} ${chunk.metadata?.symbol || ''}\n${(chunk.text || chunk.content || '').slice(0, 1500)}`;
+    return `${chunkFilePath(chunk)} ${chunk.metadata?.symbol || ''}\n${(chunk.text || chunk.content || '').slice(0, 1500)}`;
   });
 
   return { allChunks, texts };

package/core/indexing/indexer-empty-baseline.js

@@ -0,0 +1,80 @@
+/**
+ * Establish a valid *empty* index baseline.
+ *
+ * A full or incremental index run over a repository with no indexable files
+ * used to early-exit without creating anything, leaving search to throw
+ * "No search indexes found" and giving the default-on reconcile maintainer no
+ * baseline to grow from. This helper instead writes a coherent zero-row
+ * baseline:
+ *
+ *   - codebase.db            vector schema, 0 rows
+ *   - code-graph.db          graph schema, 0 rows
+ *   - merkle-state.json      0 files  — so the maintainer's dirty-scan treats
+ *                                       the first created file as new
+ *   - reconcile-manifest.json         — so readers pin a real epoch
+ *
+ * With the baseline in place, search returns empty results cleanly (the
+ * graph+codebase existence check in SweetSearch.init passes; the tables are
+ * simply empty) and the reconcile maintainer can transition the repo from zero
+ * files to one file without a prior full index.
+ *
+ * The schema builders are the same ones the production reconciler uses when it
+ * lazily creates these DBs (createVectorSchema / createGraphSchema), so a
+ * baseline written here is byte-for-byte compatible with later incremental
+ * deltas (epoch columns, FTS5, indexes).
+ */
+
+import Database from 'better-sqlite3';
+import { existsSync, mkdirSync } from 'node:fs';
+import path from 'node:path';
+
+import { DB_PATHS } from '../infrastructure/config/index.js';
+import { createVectorSchema } from './indexer-build.js';
+import { createGraphSchema } from '../graph/graph-extractor.js';
+import { publishIndexerManifest } from './indexer-manifest.js';
+import { updateState } from './incremental-tracker.js';
+import { log } from './indexer-utils.js';
+
+/**
+ * Create `dbPath` with `createSchema` only when it does not already exist.
+ * Returns true when a fresh DB was created, false when one was already present.
+ */
+function ensureSchema(dbPath, createSchema) {
+  if (existsSync(dbPath)) return false;
+  mkdirSync(path.dirname(dbPath), { recursive: true });
+  const db = new Database(dbPath);
+  try {
+    createSchema(db);
+  } finally {
+    db.close();
+  }
+  return true;
+}
+
+/**
+ * Write the empty baseline for a genuinely un-indexed empty repo.
+ *
+ * No-op when `merkle-state.json` already exists: a prior index ran, so an empty
+ * working tree means the repo BECAME empty (every tracked file deleted). In that
+ * case the existing merkle must be preserved so the maintainer's deletion
+ * detection (dirty-scan: merkle-known vs on-disk) retires the now-stale rows —
+ * overwriting it with an empty file set here would erase that knowledge and
+ * strand the stale rows in codebase.db / code-graph.db forever.
+ *
+ * @returns {Promise<{createdCodebase:boolean, createdGraph:boolean, skipped?:boolean}>}
+ */
+export async function establishEmptyBaseline() {
+  if (existsSync(DB_PATHS.merkle)) {
+    return { createdCodebase: false, createdGraph: false, skipped: true };
+  }
+  const createdCodebase = ensureSchema(DB_PATHS.codebase, createVectorSchema);
+  const createdGraph = ensureSchema(DB_PATHS.codeGraph, createGraphSchema);
+  await updateState({}, { totalChunks: 0, entities: 0, relationships: 0 });
+  publishIndexerManifest({});
+  log(
+    `Established empty index baseline (0 files; codebase.db ${createdCodebase ? 'created' : 'present'}, `
+    + `code-graph.db ${createdGraph ? 'created' : 'present'})`,
+    'green',
+  );
+  return { createdCodebase, createdGraph };
+}