ultracode 5.4.0 → 5.6.0

package/dist/chunks/chunk-B3H5NS3I.js

@@ -0,0 +1,656 @@
+import { getGpuClient } from './chunk-OMXOLFDN.js';
+import { simdL2Normalize } from './chunk-QN237S7C.js';
+import { detectBaseBranch, loadBaseMetadata, createInitialBaseMetadata, updateBaseMetadata } from './chunk-YS75L3ZS.js';
+import { init_storage_paths, normalizeBranchName, getProjectDir } from './chunk-XK2NY7RB.js';
+import { tryGarbageCollect } from './chunk-IMQ6WSJV.js';
+import { init_logging, log, logMemory } from './chunk-VCCBEJQ5.js';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { dirname, join } from 'path';
+
+// src/semantic/faiss/layered-faiss-provider.ts
+init_logging();
+init_storage_paths();
+var DEFAULT_CONFIG = {
+  dimensions: 384,
+  indexType: "hnsw",
+  hnswM: 32,
+  hnswEfConstruction: 200,
+  hnswEfSearch: 64,
+  autoSaveThreshold: 5e4
+  // Increased to reduce blocking during indexing
+};
+function getLayeredPaths(projectDir, branchName, baseBranch) {
+  const safeBranch = normalizeBranchName(branchName);
+  const safeBase = baseBranch ? normalizeBranchName(baseBranch) : safeBranch;
+  return {
+    // Base layer - uses base branch name (e.g., faiss-dev.bin for base branch "dev")
+    baseIndex: join(projectDir, `faiss-${safeBase}.bin`),
+    baseIds: join(projectDir, `faiss-${safeBase}.bin.ids.json`),
+    baseMeta: join(projectDir, "faiss-base.meta.json"),
+    // Delta layer (per feature branch)
+    deltaIndex: join(projectDir, `faiss-${safeBranch}.delta.bin`),
+    deltaIds: join(projectDir, `faiss-${safeBranch}.delta.ids.json`),
+    deltaMeta: join(projectDir, `faiss-${safeBranch}.delta.meta.json`),
+    tombstones: join(projectDir, `faiss-${safeBranch}.tombstones.json`)
+  };
+}
+var LayeredFaissProvider = class _LayeredFaissProvider {
+  config;
+  client = null;
+  isInitialized = false;
+  // Project context
+  projectPath = "";
+  projectHash = "";
+  currentBranch = null;
+  baseBranch = null;
+  // Layer state
+  isOnBaseBranch = false;
+  baseIdSet = /* @__PURE__ */ new Set();
+  deltaIdSet = /* @__PURE__ */ new Set();
+  tombstones = /* @__PURE__ */ new Set();
+  // Change tracking
+  baseUnsavedCount = 0;
+  deltaUnsavedCount = 0;
+  /** Maximum delta size before auto-merge to prevent memory leaks */
+  static MAX_DELTA_SIZE = 5e3;
+  // Initialization mutex
+  initializePromise = null;
+  constructor(config = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+  // ===========================================================================
+  // Public API
+  // ===========================================================================
+  /**
+   * Check if the provider is initialized
+   */
+  get initialized() {
+    return this.isInitialized;
+  }
+  /**
+   * Get current configured dimensions
+   */
+  getDimensions() {
+    return this.config.dimensions;
+  }
+  /**
+   * Update dimensions (e.g. when embedding model changes)
+   * Only effective before initialize() creates the FAISS index.
+   */
+  setDimensions(dimensions) {
+    if (this.config.dimensions !== dimensions) {
+      log.i("LAYERED_FAISS", "dimensions_updated", { from: this.config.dimensions, to: dimensions });
+      this.config.dimensions = dimensions;
+    }
+  }
+  // ===========================================================================
+  // Lifecycle
+  // ===========================================================================
+  /**
+   * Initialize the provider for a specific project and branch
+   * v6: CRITICAL FIX - Check if projectHash changed and reinitialize if needed!
+   * Without this, switching projects would continue using the first project's index.
+   */
+  async initialize(projectPath, projectHash, branchName) {
+    if (this.initializePromise && this.projectHash === projectHash) {
+      return this.initializePromise;
+    }
+    if (this.projectHash && this.projectHash !== projectHash) {
+      log.i("LAYERED_FAISS", "project_switch", {
+        from: this.projectHash,
+        to: projectHash
+      });
+      if (this.isInitialized) {
+        log.i("LAYERED_FAISS", "saving_before_switch", {
+          project: this.projectHash,
+          baseUnsaved: this.baseUnsavedCount,
+          deltaUnsaved: this.deltaUnsavedCount
+        });
+        await this.save();
+      }
+      this.isInitialized = false;
+      this.initializePromise = null;
+    }
+    this.initializePromise = this.initializeInternal(projectPath, projectHash, branchName);
+    return this.initializePromise;
+  }
+  async initializeInternal(projectPath, projectHash, branchName) {
+    try {
+      this.projectPath = projectPath;
+      this.projectHash = projectHash;
+      this.currentBranch = normalizeBranchName(branchName);
+      this.baseIdSet.clear();
+      this.deltaIdSet.clear();
+      this.tombstones.clear();
+      this.baseUnsavedCount = 0;
+      this.deltaUnsavedCount = 0;
+      const projectDir = getProjectDir(projectPath);
+      if (!existsSync(projectDir)) {
+        mkdirSync(projectDir, { recursive: true });
+      }
+      this.client = getGpuClient();
+      const started = await this.client.start();
+      if (!started) {
+        log.e("LAYERED_FAISS", "client_start_fail");
+        return false;
+      }
+      this.baseBranch = detectBaseBranch(projectPath);
+      this.isOnBaseBranch = this.baseBranch === this.currentBranch;
+      log.i("LAYERED_FAISS", "init", {
+        project: projectHash,
+        current: this.currentBranch,
+        base: this.baseBranch,
+        isOnBase: this.isOnBaseBranch
+      });
+      await this.loadLayers();
+      this.isInitialized = true;
+      return true;
+    } catch (error) {
+      log.e("LAYERED_FAISS", "init_fail", { err: String(error) });
+      this.initializePromise = null;
+      return false;
+    }
+  }
+  /**
+   * Load the appropriate layers based on current branch
+   */
+  async loadLayers() {
+    const projectDir = getProjectDir(this.projectPath);
+    const paths = getLayeredPaths(projectDir, this.currentBranch, this.baseBranch);
+    log.d("LAYERED_FAISS", "load_layers", {
+      projectDir,
+      projectPath: this.projectPath,
+      currentBranch: this.currentBranch,
+      baseBranch: this.baseBranch,
+      baseIndexPath: paths.baseIndex,
+      baseIndexExists: existsSync(paths.baseIndex),
+      baseIdsPath: paths.baseIds,
+      baseIdsExists: existsSync(paths.baseIds)
+    });
+    await this.loadBaseLayer(paths);
+    if (!this.isOnBaseBranch) {
+      await this.loadDeltaLayer(paths);
+    }
+  }
+  /**
+   * Load base layer index
+   */
+  async loadBaseLayer(paths) {
+    const indexConfig = {
+      dimensions: this.config.dimensions,
+      indexType: this.config.indexType,
+      metric: "l2",
+      hnswM: this.config.hnswM,
+      hnswEfConstruction: this.config.hnswEfConstruction,
+      hnswEfSearch: this.config.hnswEfSearch
+    };
+    const baseIndexExists = existsSync(paths.baseIndex);
+    const loadPath = baseIndexExists ? paths.baseIndex : void 0;
+    await this.client.faissInitialize(indexConfig, loadPath);
+    if (baseIndexExists && existsSync(paths.baseIds)) {
+      try {
+        const data = readFileSync(paths.baseIds, "utf-8");
+        const ids = JSON.parse(data);
+        this.baseIdSet = new Set(ids);
+        log.d("LAYERED_FAISS", "base_ids_loaded", { count: this.baseIdSet.size });
+      } catch (error) {
+        log.w("LAYERED_FAISS", "base_ids_load_fail", { err: String(error) });
+      }
+    }
+    const stats = await this.client.faissGetStats();
+    log.i("LAYERED_FAISS", "base_loaded", {
+      vectors: stats.totalVectors,
+      ids: this.baseIdSet.size,
+      fromFile: baseIndexExists
+    });
+  }
+  /**
+   * Load delta layer for feature branch
+   */
+  async loadDeltaLayer(paths) {
+    if (existsSync(paths.tombstones)) {
+      try {
+        const data = readFileSync(paths.tombstones, "utf-8");
+        const tombstoneList = JSON.parse(data);
+        this.tombstones = new Set(tombstoneList);
+        log.d("LAYERED_FAISS", "tombstones_loaded", { count: this.tombstones.size });
+      } catch (error) {
+        log.w("LAYERED_FAISS", "tombstones_load_fail", { err: String(error) });
+      }
+    }
+    if (existsSync(paths.deltaIds)) {
+      try {
+        const data = readFileSync(paths.deltaIds, "utf-8");
+        const ids = JSON.parse(data);
+        this.deltaIdSet = new Set(ids);
+        log.d("LAYERED_FAISS", "delta_ids_loaded", { count: this.deltaIdSet.size });
+      } catch (error) {
+        log.w("LAYERED_FAISS", "delta_ids_load_fail", { err: String(error) });
+      }
+    }
+  }
+  // ===========================================================================
+  // Branch switching
+  // ===========================================================================
+  /**
+   * Switch to a different branch
+   */
+  async switchBranch(branchName) {
+    const normalizedBranch = normalizeBranchName(branchName);
+    if (this.currentBranch === normalizedBranch) {
+      return;
+    }
+    log.i("LAYERED_FAISS", "branch_switch", {
+      from: this.currentBranch,
+      to: normalizedBranch
+    });
+    await this.save();
+    this.currentBranch = normalizedBranch;
+    this.isOnBaseBranch = this.baseBranch === normalizedBranch;
+    this.deltaIdSet.clear();
+    this.tombstones.clear();
+    this.deltaUnsavedCount = 0;
+    if (!this.isOnBaseBranch) {
+      const projectDir = getProjectDir(this.projectPath);
+      const paths = getLayeredPaths(projectDir, normalizedBranch, this.baseBranch);
+      await this.loadDeltaLayer(paths);
+    }
+    log.i("LAYERED_FAISS", "branch_switched", {
+      branch: normalizedBranch,
+      isOnBase: this.isOnBaseBranch,
+      deltaIds: this.deltaIdSet.size,
+      tombstones: this.tombstones.size
+    });
+  }
+  // ===========================================================================
+  // Search
+  // ===========================================================================
+  /**
+   * Search across all layers
+   *
+   * Algorithm:
+   * 1. Search base index for top K*2 results
+   * 2. Filter out tombstones
+   * 3. If on feature branch, search delta for top K*2
+   * 4. Merge and deduplicate (delta takes precedence)
+   * 5. Sort by similarity and return top K
+   */
+  async search(queryVector, limit) {
+    if (!this.isInitialized || !this.client) {
+      log.w("LAYERED_FAISS", "search called but not initialized", {
+        isInit: this.isInitialized,
+        hasClient: !!this.client
+      });
+      return [];
+    }
+    const normalizedQuery = simdL2Normalize(queryVector);
+    log.d("LAYERED_FAISS", "search", {
+      inputLen: queryVector.length,
+      normalizedLen: normalizedQuery.length,
+      limit,
+      normalizedSample: Array.from(normalizedQuery.slice(0, 3))
+    });
+    const searchLimit = Math.min(limit * 2, 200);
+    const results = [];
+    const seenIds = /* @__PURE__ */ new Set();
+    if (!this.isOnBaseBranch && this.deltaIdSet.size > 0) ;
+    log.d("LAYERED_FAISS", "calling faissSearch", {
+      vectorLen: normalizedQuery.length,
+      vectorType: normalizedQuery.constructor.name,
+      searchLimit
+    });
+    const baseResults = await this.client.faissSearch(normalizedQuery, searchLimit);
+    for (const result of baseResults) {
+      if (seenIds.has(result.id)) continue;
+      if (this.tombstones.has(result.id)) continue;
+      if (this.deltaIdSet.has(result.id)) {
+        continue;
+      }
+      results.push({
+        id: result.id,
+        similarity: 1 - result.distance,
+        // Convert L2 distance to similarity
+        source: "base"
+      });
+      seenIds.add(result.id);
+    }
+    for (const id of this.deltaIdSet) {
+      if (!seenIds.has(id) && results.length < limit) {
+        const baseResult = baseResults.find((r) => r.id === id);
+        if (baseResult) {
+          results.push({
+            id: baseResult.id,
+            similarity: 1 - baseResult.distance,
+            source: "delta"
+          });
+          seenIds.add(id);
+        }
+      }
+    }
+    return results.sort((a, b) => b.similarity - a.similarity).slice(0, limit);
+  }
+  /**
+   * Search and return in VectorStore-compatible format
+   */
+  async searchForVectorStore(queryVector, limit) {
+    const results = await this.search(queryVector, limit);
+    return results.map((r) => ({
+      id: r.id,
+      content: "",
+      // Content is not stored in FAISS, retrieved separately
+      similarity: r.similarity,
+      metadata: { source: r.source }
+    }));
+  }
+  // ===========================================================================
+  // Add / Remove
+  // ===========================================================================
+  /**
+   * Add a vector embedding
+   */
+  async add(embedding) {
+    if (!this.isInitialized || !this.client) {
+      return { success: false, target: "base", error: "Not initialized" };
+    }
+    const normalizedVector = simdL2Normalize(embedding.vector);
+    if (this.isOnBaseBranch) {
+      await this.client.faissAdd([embedding.id], normalizedVector);
+      this.baseIdSet.add(embedding.id);
+      this.baseUnsavedCount++;
+      if (this.baseUnsavedCount >= this.config.autoSaveThreshold) {
+        await this.saveBase();
+      }
+      return { success: true, target: "base" };
+    } else {
+      this.tombstones.delete(embedding.id);
+      this.deltaIdSet.add(embedding.id);
+      this.deltaUnsavedCount++;
+      log.i("LAYERED_FAISS", "add_to_delta", {
+        id: embedding.id.slice(0, 50),
+        branch: this.currentBranch,
+        deltaUnsaved: this.deltaUnsavedCount
+      });
+      await this.client.faissAdd([embedding.id], normalizedVector);
+      if (this.deltaUnsavedCount >= this.config.autoSaveThreshold) {
+        await this.saveDelta();
+      }
+      return { success: true, target: "delta" };
+    }
+  }
+  /**
+   * Add multiple embeddings in batch (optimized - single Named Pipe call)
+   */
+  async addBatch(embeddings) {
+    if (embeddings.length === 0) {
+      return [];
+    }
+    if (!this.isInitialized || !this.client) {
+      return embeddings.map(() => ({ success: false, target: "base", error: "Not initialized" }));
+    }
+    log.d("LAYERED_FAISS", "addBatch_called", {
+      count: embeddings.length,
+      branch: this.currentBranch,
+      isOnBase: this.isOnBaseBranch
+    });
+    const startTime = performance.now();
+    const allIds = [];
+    const dimensions = this.config.dimensions;
+    const allVectors = new Float32Array(embeddings.length * dimensions);
+    for (let i = 0; i < embeddings.length; i++) {
+      const embedding = embeddings[i];
+      const normalized = simdL2Normalize(embedding.vector);
+      allIds.push(embedding.id);
+      allVectors.set(normalized, i * dimensions);
+      if (this.isOnBaseBranch) {
+        this.baseIdSet.add(embedding.id);
+      } else {
+        this.tombstones.delete(embedding.id);
+        this.deltaIdSet.add(embedding.id);
+      }
+    }
+    await this.client.faissAdd(allIds, allVectors);
+    if (this.isOnBaseBranch) {
+      this.baseUnsavedCount += embeddings.length;
+      if (this.baseUnsavedCount >= this.config.autoSaveThreshold) {
+        await this.saveBase();
+      }
+    } else {
+      this.deltaUnsavedCount += embeddings.length;
+      if (this.deltaUnsavedCount >= this.config.autoSaveThreshold) {
+        await this.saveDelta();
+      }
+    }
+    const elapsed = performance.now() - startTime;
+    log.i("LAYERED_FAISS", "addBatch_complete", {
+      count: embeddings.length,
+      elapsed: `${elapsed.toFixed(1)}ms`,
+      speed: `${Math.round(embeddings.length / (elapsed / 1e3))}/s`,
+      target: this.isOnBaseBranch ? "base" : "delta",
+      deltaIdSetSize: this.deltaIdSet.size,
+      tombstonesSize: this.tombstones.size
+    });
+    if (!this.isOnBaseBranch && this.deltaIdSet.size > _LayeredFaissProvider.MAX_DELTA_SIZE) {
+      log.i("LAYERED_FAISS", "auto_merge_delta", {
+        deltaSize: this.deltaIdSet.size,
+        tombstones: this.tombstones.size,
+        threshold: _LayeredFaissProvider.MAX_DELTA_SIZE
+      });
+      logMemory("LAYERED_FAISS", { deltaSize: this.deltaIdSet.size, tombstones: this.tombstones.size });
+      await this.saveBase();
+      if (tryGarbageCollect(true)) {
+        log.d("LAYERED_FAISS", "gc_after_merge");
+      }
+    }
+    const target = this.isOnBaseBranch ? "base" : "delta";
+    return embeddings.map(() => ({ success: true, target }));
+  }
+  /**
+   * Remove vectors by IDs (IVectorProvider interface)
+   */
+  async remove(ids) {
+    for (const id of ids) {
+      await this.removeOne(id);
+    }
+  }
+  /**
+   * Remove a single vector by ID
+   */
+  async removeOne(id) {
+    if (!this.isInitialized || !this.client) {
+      return { success: false, action: "not_found" };
+    }
+    if (this.isOnBaseBranch) {
+      const existed = this.baseIdSet.has(id);
+      if (existed) {
+        await this.client.faissRemove([id]);
+        this.baseIdSet.delete(id);
+        this.baseUnsavedCount++;
+        return { success: true, action: "removed_from_base" };
+      }
+      return { success: false, action: "not_found" };
+    } else {
+      if (this.deltaIdSet.has(id)) {
+        this.deltaIdSet.delete(id);
+        this.deltaUnsavedCount++;
+        return { success: true, action: "removed_from_delta" };
+      } else if (this.baseIdSet.has(id)) {
+        this.tombstones.add(id);
+        this.deltaUnsavedCount++;
+        return { success: true, action: "added_tombstone" };
+      }
+      return { success: false, action: "not_found" };
+    }
+  }
+  /**
+   * Check if ID exists (considering tombstones)
+   */
+  has(id) {
+    if (this.tombstones.has(id)) {
+      return false;
+    }
+    return this.deltaIdSet.has(id) || this.baseIdSet.has(id);
+  }
+  /**
+   * Get existing IDs from a list (considering tombstones)
+   * Required by IVectorProvider interface
+   */
+  getExistingIds(ids) {
+    const existing = /* @__PURE__ */ new Set();
+    for (const id of ids) {
+      if (this.has(id)) {
+        existing.add(id);
+      }
+    }
+    return existing;
+  }
+  // ===========================================================================
+  // Persistence
+  // ===========================================================================
+  /**
+   * Save all layers
+   */
+  async save() {
+    log.i("LAYERED_FAISS", "save_called", {
+      branch: this.currentBranch,
+      isOnBase: this.isOnBaseBranch,
+      baseUnsaved: this.baseUnsavedCount,
+      deltaUnsaved: this.deltaUnsavedCount
+    });
+    if (this.baseUnsavedCount > 0) {
+      await this.saveBase();
+    }
+    if (this.deltaUnsavedCount > 0) {
+      await this.saveDelta();
+    } else if (!this.isOnBaseBranch) {
+      log.w("LAYERED_FAISS", "delta_not_saved_zero_unsaved", {
+        branch: this.currentBranch,
+        deltaIdSetSize: this.deltaIdSet.size
+      });
+    }
+  }
+  /**
+   * Save base layer
+   */
+  async saveBase() {
+    if (!this.client || !this.isInitialized) return;
+    const projectDir = getProjectDir(this.projectPath);
+    const paths = getLayeredPaths(projectDir, this.currentBranch, this.baseBranch);
+    try {
+      const dir = dirname(paths.baseIndex);
+      if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+      }
+      await this.client.faissSave(paths.baseIndex);
+      writeFileSync(paths.baseIds, JSON.stringify([...this.baseIdSet]), "utf-8");
+      const existingMeta = loadBaseMetadata(this.projectPath);
+      if (!existingMeta) {
+        createInitialBaseMetadata(this.projectPath, this.currentBranch, this.config.dimensions, this.baseIdSet.size);
+        this.baseBranch = this.currentBranch;
+        log.i("LAYERED_FAISS", "base_meta_created", { branch: this.currentBranch });
+      } else {
+        updateBaseMetadata(this.projectPath, this.baseIdSet.size);
+      }
+      this.baseUnsavedCount = 0;
+      log.i("LAYERED_FAISS", "base_saved", { vectors: this.baseIdSet.size });
+    } catch (error) {
+      log.e("LAYERED_FAISS", "base_save_fail", { err: String(error) });
+    }
+  }
+  /**
+   * Save delta layer
+   */
+  async saveDelta() {
+    if (!this.isInitialized || this.isOnBaseBranch || !this.client) return;
+    const projectDir = getProjectDir(this.projectPath);
+    const paths = getLayeredPaths(projectDir, this.currentBranch, this.baseBranch);
+    try {
+      const dir = dirname(paths.deltaIds);
+      if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+      }
+      await this.client.faissSave(paths.deltaIndex || paths.baseIndex);
+      writeFileSync(paths.deltaIds, JSON.stringify([...this.deltaIdSet]), "utf-8");
+      writeFileSync(paths.tombstones, JSON.stringify([...this.tombstones]), "utf-8");
+      const deltaMeta = {
+        branchName: this.currentBranch,
+        baseBranch: this.baseBranch,
+        deltaVectorCount: this.deltaIdSet.size,
+        tombstoneCount: this.tombstones.size,
+        createdAt: Date.now(),
+        updatedAt: Date.now()
+      };
+      writeFileSync(paths.deltaMeta, JSON.stringify(deltaMeta, null, 2), "utf-8");
+      this.deltaUnsavedCount = 0;
+      log.i("LAYERED_FAISS", "delta_saved", {
+        branch: this.currentBranch,
+        deltaIds: this.deltaIdSet.size,
+        tombstones: this.tombstones.size
+      });
+    } catch (error) {
+      log.e("LAYERED_FAISS", "delta_save_fail", { err: String(error) });
+    }
+  }
+  // ===========================================================================
+  // Statistics
+  // ===========================================================================
+  /**
+   * Get statistics about the layered index
+   */
+  async getStats() {
+    const baseVectors = this.baseIdSet.size;
+    const deltaVectors = this.deltaIdSet.size;
+    const tombstoneCount = this.tombstones.size;
+    const totalVectors = baseVectors + deltaVectors - tombstoneCount;
+    return {
+      totalVectors: Math.max(0, totalVectors),
+      baseVectors,
+      deltaVectors,
+      tombstones: tombstoneCount,
+      currentBranch: this.currentBranch,
+      baseBranch: this.baseBranch,
+      isOnBaseBranch: this.isOnBaseBranch
+    };
+  }
+  /**
+   * Get count of searchable vectors
+   */
+  async count() {
+    const stats = await this.getStats();
+    return stats.totalVectors;
+  }
+  // ===========================================================================
+  // Cleanup
+  // ===========================================================================
+  /**
+   * Close and save all state
+   */
+  async close() {
+    if (!this.isInitialized) return;
+    await this.save();
+    this.isInitialized = false;
+    this.initializePromise = null;
+    log.i("LAYERED_FAISS", "closed", {
+      project: this.projectHash,
+      branch: this.currentBranch
+    });
+  }
+};
+var layeredProviderInstance = null;
+function getLayeredFaissProvider(config) {
+  if (!layeredProviderInstance) {
+    layeredProviderInstance = new LayeredFaissProvider(config);
+  } else if (config?.dimensions && config.dimensions !== layeredProviderInstance.getDimensions()) {
+    layeredProviderInstance.setDimensions(config.dimensions);
+  }
+  return layeredProviderInstance;
+}
+async function shutdownLayeredFaissProvider() {
+  if (layeredProviderInstance) {
+    await layeredProviderInstance.close();
+    layeredProviderInstance = null;
+  }
+}
+
+export { LayeredFaissProvider, getLayeredFaissProvider, shutdownLayeredFaissProvider };
+//# sourceMappingURL=chunk-B3H5NS3I.js.map
+//# sourceMappingURL=chunk-B3H5NS3I.js.map

package/dist/chunks/chunk-DPTZHDST.js

@@ -0,0 +1,4 @@
+import {a as a$2,b as b$1}from'./chunk-IFKMF76K.js';import {I as I$1,a as a$1}from'./chunk-SV3WKUNV.js';import {b as b$2}from'./chunk-AIZJT6AG.js';import {e,d as d$1}from'./chunk-OG62YNVF.js';import {R,I as I$2}from'./chunk-4BI7EBPQ.js';import {a}from'./chunk-DXB73IDG.js';import {existsSync}from'fs';import {dirname,join}from'path';import {fileURLToPath}from'url';import {connect}from'net';R();I$1();e();R();var _={cudaBatchCosine:{dim384:2e3,dim768:1e3,dim1024:500,dim8192:50},cudaNormalize:{dim384:5e3,dim768:2e3,dim1024:1e3,dim8192:100},faissIndexBuild:{bulkInsertThreshold:500,rebuildThreshold:1e3,incrementalDeltaPercent:30},faissSearch:{minIndexSize:100,batchQueryThreshold:10}},l={..._};function A(n,e){return e<=384?n.dim384:e<=768?n.dim768:e<=1024?n.dim1024:n.dim8192}function b(n,e){let s=A(l.cudaBatchCosine,e),t=n>=s;return process.env.ADAPTIVE_DEBUG==="true"&&I$2.d("ADAPTIVE","cudaBatchCosine decision",{count:n,dim:e,threshold:s,useCuda:t}),t}function B(n,e){let s=A(l.cudaNormalize,e);return n>=s}function J(n,e){return n>=l.faissIndexBuild.rebuildThreshold||e!==void 0}function $(n,e=1){return n>=l.faissSearch.minIndexSize||e>=l.faissSearch.batchQueryThreshold}function E(n,e,s,t=1){let r=[],i="libsql-diskann";s&&J(n)?(i="faiss-hnsw",r.push(`Faiss HNSW for rebuild (${n} vectors)`)):n>1e4&&(i="faiss-ivf",r.push(`Faiss IVF for large index (${n} vectors)`));let o="libsql";$(n,t)&&(o=t>1?"faiss":"hybrid",r.push(`Faiss search (index=${n}, queries=${t})`));let c=b(n,e);return c&&r.push(`CUDA reranking (${n} \xD7 ${e}-dim)`),{indexStrategy:i,searchStrategy:o,useCudaForReranking:c,reason:r.join("; ")||"Default CPU strategy"}}var H=process.platform==="win32"?"\\\\.\\pipe\\ultracode-gpu-":"/tmp/ultracode-gpu-",P=class{buffer;length=0;initialSize;constructor(e=65536){this.initialSize=e,this.buffer=Buffer.allocUnsafe(e);}ensureCapacity(e){let s=this.length+e;if(s>this.buffer.length){let t=Math.max(this.buffer.length*2,s),r=Buffer.allocUnsafe(t);this.buffer.copy(r,0,0,this.length),this.buffer=r;}}shrinkIfEmpty(){this.length===0&&this.buffer.length>this.initialSize&&(this.buffer=Buffer.allocUnsafe(this.initialSize));}append(e){this.ensureCapacity(e.length),e.copy(this.buffer,this.length),this.length+=e.length;}get dataLength(){return this.length}readUInt32LE(e){return this.buffer.readUInt32LE(e)}subarray(e,s){return this.buffer.subarray(e,s)}consume(e){e>=this.length?this.length=0:(this.buffer.copy(this.buffer,0,e,this.length),this.length-=e),this.shrinkIfEmpty();}reset(){this.length=0,this.shrinkIfEmpty();}};var y=class{socket=null;options;pipePath;connected=false;pendingRequest=null;responseBuffer=new P(65536);constructor(e){this.options={timeout:3e4,...e},this.pipePath=H+e.pipeId;}get path(){return this.pipePath}get isConnected(){return this.connected}async connect(){if(!this.connected)return new Promise((e,s)=>{this.socket=connect(this.pipePath),this.socket.on("connect",()=>{this.connected=true,this.options.onConnect?.(),e();}),this.socket.on("data",t=>{this.handleData(t);}),this.socket.on("error",t=>{this.connected=false,this.options.onError?.(t),this.pendingRequest&&(this.pendingRequest.reject(t),this.pendingRequest=null),s(t);}),this.socket.on("close",()=>{this.connected=false,this.options.onDisconnect?.(),this.pendingRequest&&(this.pendingRequest.reject(new Error("Connection closed")),this.pendingRequest=null);});})}handleData(e){if(this.responseBuffer.append(e),this.responseBuffer.dataLength>=4){let s=this.responseBuffer.readUInt32LE(0);if(this.responseBuffer.dataLength>=4+s){let t=Buffer.from(this.responseBuffer.subarray(4,4+s));this.responseBuffer.consume(4+s),this.pendingRequest&&(this.pendingRequest.resolve(t),this.pendingRequest=null);}}}async send(e){if(!this.socket||!this.connected)throw new Error("Not connected");if(this.pendingRequest)throw new Error("Another request is pending");let s=new AbortController,t=new Promise((i,o)=>{this.pendingRequest={resolve:u=>{s.abort(),i(u);},reject:u=>{s.abort(),o(u);}};let c=Buffer.allocUnsafe(4+e.length);c.writeUInt32LE(e.length,0),e.copy(c,4),this.socket.write(c);}),r=(async()=>{if(await b$2(this.options.timeout??3e4),s.signal.aborted)return new Promise(()=>{});throw this.pendingRequest=null,new Error("Request timeout")})();return Promise.race([t,r])}disconnect(){this.socket&&(this.socket.destroy(),this.socket=null,this.connected=false);}};function G(n,e){let s=JSON.stringify(n),t=Buffer.from(s);if(!e||e.length===0){let o=Buffer.allocUnsafe(4+t.length);return o.writeUInt32LE(t.length,0),t.copy(o,4),o}let r=Buffer.from(e.buffer,e.byteOffset,e.byteLength),i=Buffer.allocUnsafe(4+t.length+r.length);return i.writeUInt32LE(t.length,0),t.copy(i,4),r.copy(i,4+t.length),i}function x(n){let e=n.readUInt32LE(0),s=n.subarray(4,4+e).toString(),t=JSON.parse(s);if(n.length>4+e){let r=4+e,i=n.length-r,o=n.subarray(r),c=new Float32Array(i/4);for(let u=0;u<c.length;u++)c[u]=o.readFloatLE(u*4);return {header:t,vectors:c}}return {header:t}}function M(n){return n.success===false&&"error"in n}function d(n){return M(n)?n.error:"Unknown GPU error"}function I(n){return n.type==="faiss.add"}function U(n){return n.type==="faiss.search"}function N(n){return n.type==="faiss.batchSearch"}function W(n){return n.type==="faiss.train"}function D(n){return n.type==="cuda.cosine"}function L(n){return n.type==="cuda.batchCosine"}function T(n){if(I(n)){let{vectors:s,...t}=n;return {vectors:s instanceof Float32Array?s:new Float32Array(s),headerData:t}}if(U(n)){let{vector:s,...t}=n;return {vectors:s instanceof Float32Array?s:new Float32Array(s),headerData:t}}if(N(n)){let{vectors:s,...t}=n;return {vectors:s instanceof Float32Array?s:new Float32Array(s),headerData:t}}if(W(n)){let{vectors:s,...t}=n;return {vectors:s instanceof Float32Array?s:new Float32Array(s),headerData:t}}if(D(n)){let{a:s,b:t,...r}=n,i=s instanceof Float32Array?s:new Float32Array(s),o=t instanceof Float32Array?t:new Float32Array(t),c=i.length+o.length,u=new Float32Array(c);u.set(i,0),u.set(o,i.length);let g={...r,dimensions:i.length};return {vectors:u,headerData:g}}if(L(n)){let{query:s,database:t,...r}=n,i=s instanceof Float32Array?s:new Float32Array(s),o=t.map(p=>p instanceof Float32Array?p:new Float32Array(p)),c=i.length+o.reduce((p,O)=>p+O.length,0),u=new Float32Array(c);u.set(i,0);let g=i.length;for(let p of o)u.set(p,g),g+=p.length;let z={...r,dimensions:i.length,queryCount:1,databaseCount:o.length};return {vectors:u,headerData:z}}return {headerData:{...n}}}var Y={nodePath:"node",workerPath:"",timeout:3e4,autoRestart:true,maxRestarts:3,forceSubprocess:false},w=class{config;worker=null;pendingRequests=new Map;requestId=0;restartCount=0;isShuttingDown=false;faissInitConfig=null;responseBuffer="";_cudaAvailable=false;namedPipeClient=null;namedPipePath=null;useNamedPipe=false;startPromise=null;namedPipeRequestQueue=Promise.resolve({});constructor(e={}){this.config={...Y,...e},this.config.workerPath||(this.config.workerPath=this.findWorkerPath()),this.registerCleanupHandlers();}registerCleanupHandlers(){let e=()=>{if(this.worker&&!this.isShuttingDown){this.isShuttingDown=true,I$2.d("GPU","Parent exiting, killing worker..."),this.namedPipeClient&&(this.namedPipeClient.disconnect(),this.namedPipeClient=null),this.worker.kill(),this.worker=null;for(let[,s]of this.pendingRequests)s.abortController.abort(),s.reject(new Error("Client shutting down"));this.pendingRequests.clear();}};process.on("exit",e),process.on("SIGINT",e),process.on("SIGTERM",e);}findWorkerPath(){let e=[],s=dirname(fileURLToPath(import.meta.url)),t=s;s.includes("chunks")?t=dirname(s):s.includes("semantic")&&(t=dirname(dirname(s))),e.push(join(t,"semantic","gpu","gpu-worker.js")),e.push(join(s,"gpu-worker.js"));try{let i=a.resolve("ultracode");e.push(join(dirname(i),"semantic/gpu/gpu-worker.js"));}catch{}e.push(join(process.cwd(),"dist/semantic/gpu/gpu-worker.js"));for(let i of e)if(existsSync(i))return I$2.d("GPU","Found worker at",{path:i}),i;let r=e[0]??join(process.cwd(),"dist/semantic/gpu/gpu-worker.js");return I$2.w("GPU","Worker not found, using fallback",{path:r,candidates:e}),r}async start(){return this.startPromise?this.startPromise:this.worker?true:process.env.ULTRACODE_NO_SUBPROCESS==="1"?(I$2.d("GPU","SKIPPED (ULTRACODE_NO_SUBPROCESS=1)"),false):(this.startPromise=this.startInternal(),this.startPromise)}async startInternal(){try{I$2.i("GPU","Starting worker",{nodePath:this.config.nodePath,workerPath:this.config.workerPath});let e=process.platform==="win32";if(!(typeof Bun<"u")||e){let{spawn:t}=await import('child_process');this.worker=t(this.config.nodePath,["--expose-gc",this.config.workerPath],{stdio:["pipe","pipe","pipe"],windowsHide:!0}),this.worker.unref(),this.worker.stderr&&this.worker.stderr.on("data",r=>{let i=r.toString().trim();i&&I$2.d("GPU",i);});}else {let r=globalThis.Bun?.spawn([this.config.nodePath,"--expose-gc",this.config.workerPath],{stdin:"pipe",stdout:"pipe",stderr:"pipe"}),i={stdin:r.stdin,stdout:r.stdout,stderr:r.stderr,pid:r.pid,kill:()=>r.kill(),on:(o,c)=>{(o==="exit"||o==="close")&&r.exited.then(u=>c(u));}};this.worker=i;}this.setupStdoutReader(),await this.waitForReady();try{let t=await this.cudaInfo();this._cudaAvailable=t.available;}catch{}return I$2.i("GPU","Worker started",{cuda:this._cudaAvailable}),!0}catch(e){return I$2.e("GPU","Failed to start worker",{error:e.message}),false}}setupStdoutReader(){this.worker?.stdout&&(this.worker.stdout.on("data",e=>{this.responseBuffer+=e.toString(),this.processResponseBuffer();}),this.worker.stdout.on("error",e=>{this.isShuttingDown||(I$2.e("GPU","stdout read error",{error:e.message}),this.handleWorkerCrash());}),this.worker.on("exit",e=>{this.isShuttingDown||(I$2.w("GPU","Worker exited",{code:e}),this.handleWorkerCrash());}));}processResponseBuffer(){let e=this.responseBuffer.split(`
+`);this.responseBuffer=e.pop()||"";for(let s of e)if(s.trim())try{let t=JSON.parse(s);if(t.type==="pipe.ready"&&t.path){this.namedPipePath=t.path,I$2.d("GPU","Received pipe.ready",{path:t.path});continue}let r=t;this.handleResponse(r);}catch{}}handleResponse(e){let s=this.pendingRequests.entries().next().value;if(s){let[t,r]=s;r.abortController.abort(),this.pendingRequests.delete(t),r.resolve(e);}}async waitForReady(){let s=Date.now();for(;Date.now()-s<5e3&&!this.namedPipePath;)await d$1(50);if(this.namedPipePath)try{let t=this.namedPipePath.split(/[/\\]/).pop()?.replace("ultracode-gpu-","").replace(".sock","")||"";this.namedPipeClient=new y({pipeId:t,timeout:this.config.timeout,onConnect:()=>{I$2.i("GPU","Connected to Named Pipe",{path:this.namedPipePath});},onDisconnect:()=>{I$2.d("GPU","Named Pipe disconnected"),this.useNamedPipe=!1;},onError:r=>{I$2.w("GPU","Named Pipe error",{error:r.message}),this.useNamedPipe=!1;}}),await this.namedPipeClient.connect(),this.useNamedPipe=!0,I$2.i("GPU","Using Named Pipe for binary IPC");}catch(t){I$2.w("GPU","Failed to connect to Named Pipe, using stdin/stdout",{error:t.message}),this.namedPipeClient=null,this.useNamedPipe=false;}else I$2.d("GPU","Named Pipe not available, using stdin/stdout"),await d$1(100);}async handleWorkerCrash(){if(!this.isShuttingDown){for(let[,e]of this.pendingRequests)e.abortController.abort(),e.reject(new Error("Worker crashed"));this.pendingRequests.clear(),this.worker=null,this.config.autoRestart&&this.restartCount<this.config.maxRestarts&&(this.restartCount++,await this.start()&&this.faissInitConfig&&await this.faissInitialize(this.faissInitConfig));}}async stop(){if(this.isShuttingDown=true,this.namedPipeClient&&(this.namedPipeClient.disconnect(),this.namedPipeClient=null,this.useNamedPipe=false),this.worker){try{this.worker.stdin?.write(`${JSON.stringify({type:"shutdown"})}
+`),await d$1(500);}catch{}if(this.worker){try{this.worker.kill("SIGTERM");}catch{}this.worker=null;}}for(let[,e]of this.pendingRequests)e.abortController.abort(),e.reject(new Error("Client shutting down"));this.pendingRequests.clear();}isRunning(){return this.worker!==null&&!this.isShuttingDown}async sendRequest(e){if(!this.worker?.stdin)throw new Error("Worker not running");if(this.useNamedPipe&&this.namedPipeClient?.isConnected)return I$2.i("GPU","Using Named Pipe",{type:e.type}),this.sendNamedPipeRequest(e);let s=JSON.stringify(e);I$2.i("GPU","Using stdin/stdout JSON",{type:e.type,jsonLen:s.length});let t=++this.requestId,r=new AbortController,i=new Promise((c,u)=>{this.pendingRequests.set(t,{resolve:c,reject:u,abortController:r}),this.worker.stdin.write(`${s}
+`);}),o=(async()=>{if(await d$1(this.config.timeout),r.signal.aborted)return new Promise(()=>{});throw this.pendingRequests.delete(t),new Error(`Request timeout: ${e.type}`)})();return Promise.race([i,o])}async sendNamedPipeRequest(e){let t=this.namedPipeRequestQueue.catch(()=>{}).then(()=>this.sendNamedPipeRequestInternal(e));return this.namedPipeRequestQueue=t,t}async sendNamedPipeRequestInternal(e){if(!this.namedPipeClient?.isConnected)throw new Error("Named Pipe not connected");let{vectors:s,headerData:t}=T(e),r=G(t,s),i=await this.namedPipeClient.send(r),{header:o}=x(i);return o}async faissInitialize(e,s){this.faissInitConfig=e,this.worker||await this.start();let t=await this.sendRequest({type:"faiss.init",config:e,loadPath:s});if(!t.success)throw new Error(d(t));return t}async faissAdd(e,s){let t=s instanceof Float32Array?Array.from(s):s,r=await this.sendRequest({type:"faiss.add",ids:e,vectors:t});if(!r.success)throw new Error(d(r));return r}async faissSearch(e,s){let t=e instanceof Float32Array?Array.from(e):e;I$2.i("GPU","faissSearch input",{vectorLen:t?.length,k:s,isArray:Array.isArray(t)});let r={type:"faiss.search",vector:t,k:s};I$2.i("GPU","faissSearch request",{hasVector:!!r.vector,vectorLen:r.vector?.length,keys:Object.keys(r)});let i=await this.sendRequest(r);if(!i.success)throw new Error(d(i));return i.results}async faissBatchSearch(e,s,t){let r=e instanceof Float32Array?Array.from(e):e,i=await this.sendRequest({type:"faiss.batchSearch",vectors:r,nQueries:s,k:t});if(!i.success)throw new Error(d(i));return i.results}async faissTrain(e,s){let t=e instanceof Float32Array?Array.from(e):e,r=await this.sendRequest({type:"faiss.train",vectors:t,nVectors:s});if(!r.success)throw new Error(d(r));return r}async faissSave(e){let s=e||join(a$1(),"faiss-index.bin"),t=await this.sendRequest({type:"faiss.save",path:s});if(!t.success)throw new Error(d(t));return t}async faissLoad(e){let s=await this.sendRequest({type:"faiss.load",path:e});if(!s.success)throw new Error(d(s));return s}async faissRemove(e){let s=await this.sendRequest({type:"faiss.remove",ids:e});if(!s.success)throw new Error(d(s))}async faissGetStats(){let e=await this.sendRequest({type:"faiss.stats"});if(!e.success)throw new Error(d(e));return e.stats}async cudaInfo(){let e=await this.sendRequest({type:"cuda.info"});if(!e.success)throw new Error(d(e));return e}isCudaAvailable(){return this._cudaAvailable}async cudaCosineSimilarity(e,s){let t=e instanceof Float32Array?Array.from(e):e,r=s instanceof Float32Array?Array.from(s):s,i=await this.sendRequest({type:"cuda.cosine",a:t,b:r});if(!i.success)throw new Error(d(i));return i.similarity}async cudaBatchCosineSimilarity(e,s){let t=e instanceof Float32Array?Array.from(e):e,r=s.map(o=>o instanceof Float32Array?Array.from(o):o),i=await this.sendRequest({type:"cuda.batchCosine",query:t,database:r});if(!i.success)throw new Error(d(i));return new Float32Array(i.similarities)}async cudaEuclideanDistance(e,s){let t=e instanceof Float32Array?Array.from(e):e,r=s instanceof Float32Array?Array.from(s):s,i=await this.sendRequest({type:"cuda.euclidean",a:t,b:r});if(!i.success)throw new Error(d(i));return i.distance}async cudaNormalizeVectors(e){let s=e.map(r=>r instanceof Float32Array?Array.from(r):r),t=await this.sendRequest({type:"cuda.normalize",vectors:s});if(!t.success)throw new Error(d(t));return t.vectors.map(r=>new Float32Array(r))}async adaptiveBatchCosineSimilarity(e,s){let t=s.length,r=e.length;if(this._cudaAvailable&&b(t,r))try{return {similarities:await this.cudaBatchCosineSimilarity(e,s),usedCuda:!0}}catch(o){I$2.w("GPU","CUDA batch cosine failed, falling back to CPU",{error:o.message});}let i=new Float32Array(t);for(let o=0;o<t;o++)i[o]=a$2(e,s[o]);return {similarities:i,usedCuda:false}}async adaptiveNormalizeVectors(e){let s=e.length,t=e[0]?.length??0;if(this._cudaAvailable&&B(s,t))try{return {normalized:await this.cudaNormalizeVectors(e),usedCuda:!0}}catch(i){I$2.w("GPU","CUDA normalize failed, falling back to CPU",{error:i.message});}return {normalized:e.map(i=>{let o=new Float32Array(i);return b$1(o)}),usedCuda:false}}getRecommendedStrategy(e,s,t,r=1){return E(e,s,t,r)}async getStats(){let e=await this.sendRequest({type:"stats"});if(!e.success)throw new Error(d(e));return e}},f=null;function ve(n){return f||(I$2.i("GPU","Using subprocess mode"),f=new w(n)),f}async function qe(){f&&(await f.stop(),f=null);}export{E as a,ve as b,qe as c};