ultracode 5.4.0 → 5.6.0

package/dist/chunks/chunk-S5Q7BD6J.js

@@ -0,0 +1,572 @@
+import { getGpuClient } from './chunk-OMXOLFDN.js';
+import { simdL2Normalize } from './chunk-QN237S7C.js';
+import { getBaseMetaPathByHash } from './chunk-YS75L3ZS.js';
+import { init_storage_paths, normalizeBranchName, getFaissIndexPathByHash } from './chunk-XK2NY7RB.js';
+import { init_logging, log } from './chunk-VCCBEJQ5.js';
+import { existsSync, mkdirSync, writeFileSync, readFileSync, unlinkSync } from 'fs';
+import { dirname } from 'path';
+
+// src/semantic/faiss/faiss-provider.ts
+init_logging();
+init_storage_paths();
+var DEFAULT_CONFIG = {
+  dimensions: 768,
+  indexType: "hnsw",
+  hnswM: 32,
+  hnswEfConstruction: 200,
+  hnswEfSearch: 64,
+  autoSaveThreshold: 5e3,
+  // Save after 5000 new embeddings
+  persistPath: ""
+};
+var FaissProvider = class {
+  config;
+  client = null;
+  isInitialized = false;
+  // Track unsaved changes for auto-save
+  unsavedCount = 0;
+  lastSaveTime = 0;
+  // Background save state (avoid blocking pipeline)
+  pendingSave = null;
+  saveScheduled = false;
+  // Project context (FAISS index is per-project, per-branch)
+  projectHash = "default";
+  branchName = "main";
+  // ID set for existence checks (content stored in LibSQL, not here)
+  idSet = /* @__PURE__ */ new Set();
+  // Mutex for initialize() to prevent race condition with concurrent calls
+  initializePromise = null;
+  constructor(config = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+  /**
+   * Set project context for per-project FAISS index
+   * If context changes while initialized, saves current index and loads new one
+   */
+  async setProjectContext(projectHash, branchName = "main") {
+    const normalizedBranch = normalizeBranchName(branchName);
+    const newPersistPath = getFaissIndexPathByHash(projectHash, normalizedBranch);
+    if (this.projectHash === projectHash && this.branchName === normalizedBranch) {
+      return;
+    }
+    if (this.isInitialized && this.client) {
+      log.i("FAISS", "Switching project context", {
+        from: { projectHash: this.projectHash, branchName: this.branchName },
+        to: { projectHash, branchName: normalizedBranch }
+      });
+      if (this.unsavedCount > 0) {
+        await this.save();
+      }
+      this.saveIdSet();
+      this.projectHash = projectHash;
+      this.branchName = normalizedBranch;
+      this.config.persistPath = newPersistPath;
+      this.idSet.clear();
+      this.unsavedCount = 0;
+      const loadPath = existsSync(newPersistPath) ? newPersistPath : void 0;
+      const indexConfig = {
+        dimensions: this.config.dimensions,
+        indexType: this.config.indexType,
+        metric: "l2",
+        hnswM: this.config.hnswM,
+        hnswEfConstruction: this.config.hnswEfConstruction,
+        hnswEfSearch: this.config.hnswEfSearch
+      };
+      await this.client.faissInitialize(indexConfig, loadPath);
+      if (loadPath) {
+        this.loadIdSet();
+      }
+      const stats = await this.client.faissGetStats();
+      log.i("FAISS", "Switched to new context", {
+        projectHash,
+        branchName: normalizedBranch,
+        vectors: stats.totalVectors,
+        idSetSize: this.idSet.size
+      });
+    } else {
+      this.projectHash = projectHash;
+      this.branchName = normalizedBranch;
+      this.config.persistPath = newPersistPath;
+      log.d("FAISS", "Project context set", {
+        projectHash: this.projectHash,
+        branchName: this.branchName,
+        persistPath: this.config.persistPath
+      });
+    }
+  }
+  // ===========================================================================
+  // Lifecycle
+  // ===========================================================================
+  async initialize() {
+    if (this.isInitialized) return true;
+    if (this.initializePromise) {
+      return this.initializePromise;
+    }
+    this.initializePromise = this.initializeInternal();
+    return this.initializePromise;
+  }
+  async initializeInternal() {
+    try {
+      log.i("FAISS", "Initializing...");
+      this.client = getGpuClient();
+      const started = await this.client.start();
+      if (!started) {
+        log.e("FAISS", "Failed to start Faiss client");
+        return false;
+      }
+      const indexConfig = {
+        dimensions: this.config.dimensions,
+        indexType: this.config.indexType,
+        metric: "l2",
+        hnswM: this.config.hnswM,
+        hnswEfConstruction: this.config.hnswEfConstruction,
+        hnswEfSearch: this.config.hnswEfSearch
+      };
+      let loadPath = existsSync(this.config.persistPath) ? this.config.persistPath : void 0;
+      if (loadPath) {
+        const savedMeta = this.loadIndexMetadata();
+        if (savedMeta && savedMeta.dimensions !== this.config.dimensions) {
+          log.w("FAISS", "Dimension mismatch - dropping old index", {
+            savedDimensions: savedMeta.dimensions,
+            configDimensions: this.config.dimensions,
+            indexPath: loadPath
+          });
+          this.deleteIndexFiles();
+          loadPath = void 0;
+        }
+      }
+      await this.client.faissInitialize(indexConfig, loadPath);
+      if (loadPath) {
+        this.loadIdSet();
+      }
+      this.isInitialized = true;
+      this.lastSaveTime = Date.now();
+      const stats = await this.client.faissGetStats();
+      log.i("FAISS", "Initialized", {
+        indexType: this.config.indexType,
+        dimensions: this.config.dimensions,
+        vectors: stats.totalVectors,
+        loaded: !!loadPath,
+        idSetSize: this.idSet.size
+      });
+      return true;
+    } catch (error) {
+      log.e("FAISS", "Initialization failed", { error: error.message });
+      return false;
+    }
+  }
+  async close() {
+    if (this.unsavedCount > 0) {
+      await this.save();
+    }
+    if (this.client && this.isInitialized) {
+      await this.client.stop();
+    }
+    this.idSet.clear();
+    this.isInitialized = false;
+  }
+  // ===========================================================================
+  // Insert Operations
+  // ===========================================================================
+  /**
+   * Add single embedding to Faiss
+   */
+  async add(embedding) {
+    if (!this.client || !this.isInitialized) {
+      throw new Error("FaissProvider not initialized");
+    }
+    await this.client.faissAdd([embedding.id], Array.from(embedding.vector));
+    this.idSet.add(embedding.id);
+    this.unsavedCount++;
+    if (this.unsavedCount >= this.config.autoSaveThreshold) {
+      await this.save();
+    }
+  }
+  /**
+   * Batch add embeddings to Faiss
+   */
+  async addBatch(embeddings) {
+    if (!this.client || !this.isInitialized) {
+      throw new Error("FaissProvider not initialized");
+    }
+    if (embeddings.length === 0) return;
+    const pStart = performance.now();
+    const pLog = (phase) => {
+      const elapsed = (performance.now() - pStart).toFixed(1);
+      log.i("FAISS", phase, { elapsedMs: elapsed });
+    };
+    const dim = this.config.dimensions;
+    const count = embeddings.length;
+    const ids = new Array(count);
+    const vectors = new Float32Array(count * dim);
+    for (let i = 0; i < count; i++) {
+      const emb = embeddings[i];
+      ids[i] = emb.id;
+      const normalized = simdL2Normalize(emb.vector);
+      vectors.set(normalized, i * dim);
+      this.idSet.add(emb.id);
+    }
+    pLog("F1_PREPARE_VECTORS");
+    await this.client.faissAdd(ids, vectors);
+    pLog("F2_FAISS_ADD");
+    this.unsavedCount += count;
+    log.d("FAISS", "Added embeddings", {
+      count,
+      unsaved: this.unsavedCount
+    });
+    if (this.unsavedCount >= this.config.autoSaveThreshold && !this.saveScheduled) {
+      this.saveScheduled = true;
+      this.pendingSave = this.save().catch((err) => log.w("FAISS", "Background save failed", { error: err.message })).finally(() => {
+        this.saveScheduled = false;
+        this.pendingSave = null;
+      });
+    }
+  }
+  /**
+   * Wait for any pending background save to complete
+   */
+  async waitForPendingSave() {
+    if (this.pendingSave) {
+      await this.pendingSave;
+    }
+  }
+  // ===========================================================================
+  // Search Operations
+  // ===========================================================================
+  /**
+   * Search for similar vectors in Faiss
+   * Returns only id + score. Content must be fetched from LibSQL separately.
+   */
+  async search(queryVector, limit = 10) {
+    if (!this.client || !this.isInitialized) {
+      throw new Error("FaissProvider not initialized");
+    }
+    const normalizedQuery = simdL2Normalize(queryVector);
+    const faissResults = await this.client.faissSearch(normalizedQuery, limit);
+    return faissResults.map((r) => ({
+      id: r.id,
+      similarity: r.score,
+      content: ""
+      // Content stored in LibSQL, not in Faiss
+    }));
+  }
+  /**
+   * Batch search for multiple query vectors
+   * Returns only id + score. Content must be fetched from LibSQL separately.
+   */
+  async batchSearch(queryVectors, limit = 10) {
+    if (!this.client || !this.isInitialized) {
+      throw new Error("FaissProvider not initialized");
+    }
+    const dim = this.config.dimensions;
+    const count = queryVectors.length;
+    const flatVectors = new Float32Array(count * dim);
+    for (let i = 0; i < count; i++) {
+      const normalized = simdL2Normalize(queryVectors[i]);
+      flatVectors.set(normalized, i * dim);
+    }
+    const faissResults = await this.client.faissBatchSearch(flatVectors, count, limit);
+    return faissResults.map(
+      (queryResults) => queryResults.map((r) => ({
+        id: r.id,
+        similarity: r.score,
+        content: ""
+        // Content stored in LibSQL, not in Faiss
+      }))
+    );
+  }
+  // ===========================================================================
+  // Persistence
+  // ===========================================================================
+  /**
+   * Save Faiss index and content cache to disk
+   */
+  async save() {
+    if (!this.client || !this.isInitialized) {
+      throw new Error("FaissProvider not initialized");
+    }
+    if (this.unsavedCount === 0) return;
+    try {
+      const dir = dirname(this.config.persistPath);
+      if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+      }
+      await this.client.faissSave(this.config.persistPath);
+      this.saveIdSet();
+      this.saveIndexMetadata();
+      const savedCount = this.unsavedCount;
+      this.unsavedCount = 0;
+      this.lastSaveTime = Date.now();
+      log.i("FAISS", "Index saved", {
+        path: this.config.persistPath,
+        savedCount,
+        idSetSize: this.idSet.size
+      });
+    } catch (error) {
+      log.e("FAISS", "Failed to save index", { error: error.message });
+      throw error;
+    }
+  }
+  /**
+   * Get ID set file path
+   */
+  getIdSetPath() {
+    return `${this.config.persistPath}.ids.json`;
+  }
+  /**
+   * Save ID set to disk (lightweight - just IDs, no content)
+   */
+  saveIdSet() {
+    if (!this.config.persistPath) {
+      return;
+    }
+    const idSetPath = this.getIdSetPath();
+    try {
+      const ids = Array.from(this.idSet);
+      writeFileSync(idSetPath, JSON.stringify(ids), "utf-8");
+      log.d("FAISS", "ID set saved", { path: idSetPath, size: this.idSet.size });
+    } catch (error) {
+      log.w("FAISS", "Failed to save ID set", { error: error.message });
+    }
+  }
+  /**
+   * Load ID set from disk
+   */
+  loadIdSet() {
+    const idSetPath = this.getIdSetPath();
+    if (!existsSync(idSetPath)) {
+      const legacyPath = `${this.config.persistPath}.content.json`;
+      if (existsSync(legacyPath)) {
+        try {
+          const data = readFileSync(legacyPath, "utf-8");
+          const cacheObj = JSON.parse(data);
+          this.idSet.clear();
+          for (const id of Object.keys(cacheObj)) {
+            this.idSet.add(id);
+          }
+          log.i("FAISS", "Migrated IDs from legacy content cache", { size: this.idSet.size });
+          this.saveIdSet();
+          return;
+        } catch {
+        }
+      }
+      log.d("FAISS", "No ID set file found", { path: idSetPath });
+      return;
+    }
+    try {
+      const data = readFileSync(idSetPath, "utf-8");
+      const ids = JSON.parse(data);
+      this.idSet.clear();
+      for (const id of ids) {
+        this.idSet.add(id);
+      }
+      log.i("FAISS", "ID set loaded", { path: idSetPath, size: this.idSet.size });
+    } catch (error) {
+      log.w("FAISS", "Failed to load ID set", { error: error.message });
+    }
+  }
+  /**
+   * Get index metadata file path
+   */
+  getMetadataPath() {
+    return `${this.config.persistPath}.meta.json`;
+  }
+  /**
+   * Save index metadata (dimensions, etc.)
+   * Also creates faiss-base.meta.json on first save to mark base branch
+   */
+  saveIndexMetadata() {
+    if (!this.config.persistPath) return;
+    const metaPath = this.getMetadataPath();
+    try {
+      const meta = {
+        dimensions: this.config.dimensions,
+        indexType: this.config.indexType,
+        savedAt: Date.now()
+      };
+      writeFileSync(metaPath, JSON.stringify(meta), "utf-8");
+      log.d("FAISS", "Index metadata saved", { path: metaPath, dimensions: meta.dimensions });
+      this.ensureBaseMetadata();
+    } catch (error) {
+      log.w("FAISS", "Failed to save index metadata", { error: error.message });
+    }
+  }
+  /**
+   * Ensure faiss-base.meta.json exists (create on first indexing)
+   * This marks which branch is the base for layered index support
+   */
+  ensureBaseMetadata() {
+    const baseMetaPath = getBaseMetaPathByHash(this.projectHash);
+    if (existsSync(baseMetaPath)) {
+      return;
+    }
+    try {
+      const baseMeta = {
+        baseBranch: this.branchName,
+        vectorCount: this.idSet.size,
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+        dimensions: this.config.dimensions,
+        indexType: this.config.indexType
+      };
+      const dir = dirname(baseMetaPath);
+      if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+      }
+      writeFileSync(baseMetaPath, JSON.stringify(baseMeta, null, 2), "utf-8");
+      log.i("FAISS", "Base metadata created", {
+        branch: this.branchName,
+        path: baseMetaPath,
+        vectors: this.idSet.size
+      });
+    } catch (error) {
+      log.w("FAISS", "Failed to create base metadata", { error: error.message });
+    }
+  }
+  /**
+   * Load index metadata
+   */
+  loadIndexMetadata() {
+    const metaPath = this.getMetadataPath();
+    if (!existsSync(metaPath)) {
+      return null;
+    }
+    try {
+      const data = readFileSync(metaPath, "utf-8");
+      return JSON.parse(data);
+    } catch (error) {
+      log.w("FAISS", "Failed to load index metadata", { error: error.message });
+      return null;
+    }
+  }
+  /**
+   * Delete all index files (index, ID set, metadata)
+   */
+  deleteIndexFiles() {
+    const filesToDelete = [this.config.persistPath, this.getIdSetPath(), this.getMetadataPath()];
+    for (const filePath of filesToDelete) {
+      if (filePath && existsSync(filePath)) {
+        try {
+          unlinkSync(filePath);
+          log.d("FAISS", "Deleted index file", { path: filePath });
+        } catch (error) {
+          log.w("FAISS", "Failed to delete file", { path: filePath, error: error.message });
+        }
+      }
+    }
+    this.idSet.clear();
+    this.unsavedCount = 0;
+  }
+  /**
+   * Force flush and save (compatibility method)
+   */
+  async flush() {
+    await this.waitForPendingSave();
+    const count = this.unsavedCount;
+    if (count > 0) {
+      await this.save();
+    }
+    return count;
+  }
+  /**
+   * Flush and save (alias for vector-store compatibility)
+   */
+  async flushAndSave() {
+    await this.waitForPendingSave();
+    await this.save();
+  }
+  // ===========================================================================
+  // Index Maintenance
+  // ===========================================================================
+  /**
+   * Train IVF/PQ indexes (required before adding vectors)
+   */
+  async train(trainingVectors) {
+    if (!this.client || !this.isInitialized) {
+      throw new Error("FaissProvider not initialized");
+    }
+    const flatVectors = [];
+    for (const v of trainingVectors) {
+      flatVectors.push(...Array.from(v));
+    }
+    await this.client.faissTrain(new Float32Array(flatVectors), trainingVectors.length);
+    log.i("FAISS", "Trained index", { vectorCount: trainingVectors.length });
+  }
+  // ===========================================================================
+  // Statistics
+  // ===========================================================================
+  async getStats() {
+    const faissStats = this.client ? await this.client.getStats() : null;
+    return {
+      totalVectors: faissStats?.totalVectors ?? 0,
+      unsavedCount: this.unsavedCount,
+      lastSaveTime: this.lastSaveTime,
+      idSetSize: this.idSet.size,
+      faissStats
+    };
+  }
+  isReady() {
+    return this.isInitialized && this.client?.isRunning() === true;
+  }
+  /**
+   * Get total vector count
+   */
+  async getVectorCount() {
+    const stats = await this.getStats();
+    return stats.totalVectors;
+  }
+  /**
+   * Check which IDs already exist in the ID set
+   * Used for incremental indexing to skip already-indexed entities
+   */
+  getExistingIds(ids) {
+    const existing = /* @__PURE__ */ new Set();
+    for (const id of ids) {
+      if (this.idSet.has(id)) {
+        existing.add(id);
+      }
+    }
+    return existing;
+  }
+  /**
+   * Check if an ID exists in the ID set
+   */
+  hasId(id) {
+    return this.idSet.has(id);
+  }
+  /**
+   * Remove embeddings by IDs (for re-indexing changed files)
+   */
+  async remove(ids) {
+    if (!this.client || !this.isInitialized) {
+      throw new Error("FaissProvider not initialized");
+    }
+    await this.client.faissRemove(ids);
+    for (const id of ids) {
+      this.idSet.delete(id);
+    }
+    log.d("FAISS", "Removed embeddings", { count: ids.length });
+  }
+};
+var faissProvider = null;
+function getFaissProvider(config) {
+  if (!faissProvider) {
+    faissProvider = new FaissProvider(config);
+  }
+  return faissProvider;
+}
+async function initializeFaissProvider(config) {
+  const provider = getFaissProvider(config);
+  const success = await provider.initialize();
+  return success ? provider : null;
+}
+async function shutdownFaissProvider() {
+  if (faissProvider) {
+    await faissProvider.close();
+    faissProvider = null;
+  }
+}
+
+export { FaissProvider, getFaissProvider, initializeFaissProvider, shutdownFaissProvider };
+//# sourceMappingURL=chunk-S5Q7BD6J.js.map
+//# sourceMappingURL=chunk-S5Q7BD6J.js.map