sweet-search 2.5.2 → 2.5.4

package/core/incremental-indexing/application/production-li-delta.mjs

@@ -0,0 +1,192 @@
+import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import { LateInteractionIndex } from '../../ranking/late-interaction-index.js';
+import { encodeDocumentsCpu } from '../../ranking/late-interaction-model.js';
+import {
+  openSegmentState,
+  tombstoneDoc,
+  persistSegmentState,
+  nextSegmentSeq,
+} from '../infrastructure/li-segment-state.mjs';
+
+function readJson(filePath, fallback = null) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+  } catch {
+    return fallback;
+  }
+}
+
+function segmentedState(indexPath) {
+  const stub = readJson(indexPath);
+  if (stub?.format !== 'segmented' || !stub.segmentDir) return null;
+  const segmentDir = path.resolve(path.dirname(indexPath), stub.segmentDir);
+  const manifestPath = path.join(segmentDir, 'manifest.json');
+  const manifest = readJson(manifestPath);
+  if (!manifest || !Array.isArray(manifest.segments)) return null;
+  return { segmentDir, manifestPath, manifest };
+}
+
+function manifestFromIndex(index) {
+  return {
+    version: '3.0',
+    format: 'sslx-v3',
+    modelId: index.modelId,
+    tokenDim: index.tokenDim,
+    matryoshkaDim: index.matryoshkaDim || 0,
+    maxTokens: index.maxTokens,
+    useInt8: index.useInt8,
+    quantBits: index.quantBits || (index.useInt8 ? 8 : 32),
+    poolFactor: index.poolFactor || 1,
+    whtSeed: index.whtSeed || 0,
+    whtOrdering: index.whtOrdering || 'natural',
+    totalDocuments: 0,
+    nextSeq: 0,
+    segments: [],
+  };
+}
+
+async function writeJsonAtomic(filePath, payload) {
+  await fsp.mkdir(path.dirname(filePath), { recursive: true });
+  const tmp = `${filePath}.tmp.${process.pid}`;
+  await fsp.writeFile(tmp, JSON.stringify(payload, null, 2));
+  await fsp.rename(tmp, filePath);
+}
+
+async function appendGrowingSegment(indexPath, index, docs) {
+  if (docs.size === 0) return 0;
+  const existing = segmentedState(indexPath);
+  const segmentDir = existing?.segmentDir || `${indexPath}.segments`;
+  const manifestPath = existing?.manifestPath || path.join(segmentDir, 'manifest.json');
+  const manifest = existing?.manifest || manifestFromIndex(index);
+  await fsp.mkdir(segmentDir, { recursive: true });
+
+  const seq = nextSegmentSeq(manifest);
+  const segName = `segment-${String(seq).padStart(4, '0')}.bin`;
+  const segPath = path.join(segmentDir, segName);
+  await index._writeSegmentFile(segPath, docs);
+  manifest.segments.push({ path: segName, count: docs.size });
+  manifest.nextSeq = seq + 1;
+  manifest.totalDocuments = (manifest.totalDocuments || 0) + docs.size;
+
+  await writeJsonAtomic(manifestPath, manifest);
+  await writeJsonAtomic(indexPath, {
+    version: '3.0',
+    format: 'segmented',
+    segmentDir: path.basename(segmentDir),
+  });
+  return docs.size;
+}
+
+async function rewriteLegacyIndex(index, ops, liEncoder, pickLiInput, onProgress = null) {
+  const progress = typeof onProgress === 'function'
+    ? (phase) => { onProgress(phase); }
+    : () => {};
+  let tombstone = 0;
+  for (const op of ops) {
+    if (op.retireId && index.documents.delete(op.retireId)) tombstone += 1;
+  }
+  const addOps = ops.filter((op) => op.addId && op.chunk);
+  const texts = addOps.map(({ chunk }) => pickLiInput(chunk));
+  progress('li:encode:start');
+  const tokens = texts.length > 0 ? await liEncoder(texts) : [];
+  progress('li:encode:done');
+  let appended = 0;
+  for (let i = 0; i < addOps.length; i += 1) {
+    if (!tokens[i] || tokens[i].length === 0) continue;
+    const { addId, chunk } = addOps[i];
+    await index.add(addId, tokens[i], {
+      file: chunk.file,
+      name: chunk.metadata?.symbol,
+      type: chunk.metadata?.chunk_type,
+      startLine: chunk.metadata?.line_start || null,
+      endLine: chunk.metadata?.line_end || null,
+    });
+    appended += 1;
+    if (appended % 100 === 0) progress('li:legacy-add');
+  }
+  await index.save();
+  progress('li:legacy-save');
+  return { appended, tombstone };
+}
+
+export async function applyLateInteractionDelta({
+  indexPath,
+  ops,
+  liEncoder,
+  pickLiInput,
+  onProgress = null,
+}) {
+  const progress = typeof onProgress === 'function'
+    ? (phase) => { onProgress(phase); }
+    : () => {};
+  if (!Array.isArray(ops) || ops.length === 0) {
+    return { appended: 0, tombstone: 0 };
+  }
+
+  const encode = liEncoder || ((texts) => encodeDocumentsCpu(texts));
+  const existing = fs.existsSync(indexPath);
+  const segmented = existing ? segmentedState(indexPath) : null;
+  const index = new LateInteractionIndex({ indexPath, loadExisting: true });
+  await index.init();
+  progress('li:init');
+
+  if (existing && !segmented) {
+    return rewriteLegacyIndex(index, ops, encode, pickLiInput, progress);
+  }
+
+  let tombstone = 0;
+  const states = new Map();
+  for (const op of ops) {
+    if (!op.retireId) continue;
+    const position = index._docSegmentPositions.get(op.retireId);
+    if (!position) continue;
+    if (!states.has(position.segmentPath)) {
+      const docCount = index._segments.find((s) => s.path === position.segmentPath)?.count ?? position.docIndex + 1;
+      states.set(position.segmentPath, openSegmentState(position.segmentPath, docCount));
+    }
+    tombstoneDoc(states.get(position.segmentPath), position.docIndex);
+    tombstone += 1;
+  }
+  for (const state of states.values()) persistSegmentState(state);
+  progress('li:tombstone');
+
+  const addOps = ops.filter((op) => op.addId && op.chunk);
+  const texts = addOps.map(({ chunk }) => pickLiInput(chunk));
+  progress('li:encode:start');
+  const tokens = texts.length > 0 ? await encode(texts) : [];
+  progress('li:encode:done');
+  const writer = new LateInteractionIndex({
+    indexPath,
+    loadExisting: false,
+    tokenDim: index.tokenDim,
+    maxTokens: index.maxTokens,
+    useInt8: index.useInt8,
+    quantBits: index.quantBits,
+    modelId: index.modelId,
+    poolFactor: index.poolFactor,
+    whtSeed: index.whtSeed,
+    whtOrdering: index.whtOrdering,
+    matryoshkaDim: index.matryoshkaDim,
+  });
+  await writer.init();
+  progress('li:writer-init');
+
+  for (let i = 0; i < addOps.length; i += 1) {
+    if (!tokens[i] || tokens[i].length === 0) continue;
+    const { addId, chunk } = addOps[i];
+    await writer.add(addId, tokens[i], {
+      file: chunk.file,
+      name: chunk.metadata?.symbol,
+      type: chunk.metadata?.chunk_type,
+      startLine: chunk.metadata?.line_start || null,
+      endLine: chunk.metadata?.line_end || null,
+    });
+    if ((i + 1) % 100 === 0) progress('li:add');
+  }
+
+  const appended = await appendGrowingSegment(indexPath, writer, writer._currentSegment);
+  progress('li:append-segment');
+  return { appended, tombstone };
+}

package/core/incremental-indexing/application/production-reconciler-helpers.mjs

@@ -0,0 +1,107 @@
+import { existsSync } from 'node:fs';
+import { normalizeIdentifier } from '../../graph/graph-extractor.js';
+import { FloatVectorStore, getFloatStorePath } from '../../vector-store/float-vector-store.js';
+import {
+  loadBitmap,
+  createBitmap,
+  resizeBitmap,
+  saveBitmap,
+  setBit,
+} from '../infrastructure/tombstone-bitmap.mjs';
+
+function entitySearchText(e) {
+  return [e.name, e.signature, e.doc_comment].filter(Boolean).join(' ').toLowerCase().slice(0, 1000);
+}
+
+export function insertEntity(db, e, id, epoch, hasFts) {
+  const nameAlias = normalizeIdentifier(e.name);
+  const stmt = db.prepare(`
+    INSERT INTO entities
+    (id, file_path, type, name, signature, signature_hash, doc_comment, start_line, end_line, package, parent_class, search_text, name_alias, parent_id, hierarchy_level, logical_entity_id, epoch_written, epoch_retired, stale_since)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL)
+  `);
+  stmt.run(
+    id,
+    e.file_path,
+    e.type,
+    e.name,
+    e.signature || null,
+    e.signature_hash || null,
+    e.doc_comment || null,
+    e.start_line || null,
+    e.end_line || null,
+    e.package || null,
+    e.parent_class || null,
+    entitySearchText(e),
+    nameAlias || null,
+    null,
+    ['method', 'field', 'rpc'].includes(e.type) ? 1 : 0,
+    e.id,
+    epoch,
+  );
+  if (!hasFts) return;
+  const rowid = db.prepare('SELECT rowid FROM entities WHERE id = ?').get(id)?.rowid;
+  if (!rowid) return;
+  try { db.prepare('INSERT INTO entities_fts(rowid, name, name_alias, signature, doc_comment) VALUES (?, ?, ?, ?, ?)').run(rowid, e.name, nameAlias || null, e.signature || null, e.doc_comment || null); } catch {}
+  try { db.prepare('INSERT INTO entities_trigram(rowid, name, signature) VALUES (?, ?, ?)').run(rowid, e.name, e.signature || null); } catch {}
+}
+
+export function insertRelationships(db, relationships, liveIdFor, epoch) {
+  const stmt = db.prepare(`
+    INSERT INTO relationships
+    (source_id, target_id, target_name, type, weight, context_line, full_import_path, is_static, is_wildcard, logical_relationship_id, epoch_written, epoch_retired)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)
+  `);
+  for (const r of relationships) {
+    if (!r.target_name) continue;
+    const source = liveIdFor.get(r.source_id) || r.source_id || null;
+    const logical = `${source || ''}:${r.type}:${r.target_name}:${r.context_line || ''}`;
+    try { stmt.run(source, r.target_id || null, r.target_name, r.type, r.weight || 1, r.context_line || null, r.full_import_path || null, r.is_static ? 1 : 0, r.is_wildcard ? 1 : 0, logical, epoch); } catch {}
+  }
+}
+
+export function markBinaryStale(index, id) {
+  const idx = index.idToIndex.get(id);
+  if (idx == null) return false;
+  const stalePath = index.stalePath || `${index.indexPath}.stale.bin`;
+  let bitmap = null;
+  try { bitmap = loadBitmap(stalePath); } catch {}
+  bitmap = bitmap ? resizeBitmap(bitmap, Math.max(idx + 1, index.vectors.length, 1)) : createBitmap(Math.max(idx + 1, index.vectors.length, 1));
+  setBit(bitmap, idx);
+  saveBitmap(stalePath, bitmap);
+  index.idToIndex.delete(id);
+  index.int8Vectors.delete(id);
+  index._staleBitmapCache = null;
+  return true;
+}
+
+/**
+ * Maintain the Stage 2.5 float vector store (`codebase-float-vectors.bin`) as a
+ * sidecar of the binary HNSW index. The search runtime derives its path from
+ * the binary HNSW path, loads/reloads the two together, and full indexing
+ * builds them together — so the float store is conceptually part of the binary
+ * HNSW tier, exactly like the int8 sidecar. Keeping it in lockstep here lets
+ * reconcile-created docs get true float rescoring in Stage 2.5 instead of
+ * falling back to SQLite.
+ *
+ * Skips when the store is absent but the binary HNSW already held vectors: a
+ * store built from the delta alone would be missing every baseline doc and
+ * would mis-score them. That abnormal state keeps the existing SQLite fallback
+ * until a full rebuild restores the store.
+ *
+ * @param {string} binaryHnswPath
+ * @param {object} delta
+ * @param {Array<{id: string, vector: Float32Array}>} delta.upserts
+ * @param {string[]} delta.removeIds
+ * @param {number} delta.binaryVectorsBefore  Live binary-HNSW vectors before this delta.
+ * @param {number} delta.dimension            hnswDimension to seed a fresh empty store.
+ */
+export async function maintainFloatStore(binaryHnswPath, { upserts, removeIds, binaryVectorsBefore, dimension }) {
+  if (upserts.length === 0 && removeIds.length === 0) return;
+  const floatStorePath = getFloatStorePath(binaryHnswPath);
+  if (!existsSync(floatStorePath) && binaryVectorsBefore > 0) return;
+  const store = new FloatVectorStore();
+  await store.loadOrInit(floatStorePath, dimension);
+  store.applyDelta({ upserts, removeIds });
+  await store.save(floatStorePath);
+}