unforgit 0.6.0 → 0.7.0

package/dist/{chunk-CKUDYQYP.js → chunk-TGDR7Y7T.js}

@@ -8,6 +8,7 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
 
 // ../../packages/core/dist/index.js
 import OpenAI from "openai";
+import { createHash } from "crypto";
 import OpenAI2 from "openai";
 var DEFAULT_TTL_SECONDS_BY_TYPE = {
   episodic: 30 * 24 * 60 * 60,
@@ -93,6 +94,8 @@ var LifecycleScheduler = class {
     this.runner = runner;
     this.options = options;
   }
+  runner;
+  options;
   states = /* @__PURE__ */ new Map();
   schedule(orgId, repoId) {
     const key = `${orgId}:${repoId}`;
@@ -229,17 +232,65 @@ function resolveVisibility(input) {
   }
   return { visibility: "private", suggestion: "promote" };
 }
-var EMBEDDING_MODEL = "text-embedding-3-small";
+var OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
+var OPENAI_EMBEDDING_DIMENSIONS = 1536;
+var LOCAL_EMBEDDING_MODEL = "local-hash-multilingual-v1";
+var LOCAL_EMBEDDING_DIMENSIONS = 384;
 var cachedClient = null;
 function isOpenAIConfigured(apiKey) {
   const key = apiKey ?? process.env.OPENAI_API_KEY;
-  return !!key && key !== "sk-your-api-key-here" && key.startsWith("sk-");
+  return !!key && key !== "sk-you...here" && key.startsWith("sk-");
+}
+function resolveEmbeddingProvider(config2) {
+  const requested = config2?.provider ?? "auto";
+  const openaiAvailable = isOpenAIConfigured(config2?.apiKey);
+  if (requested === "disabled") {
+    return {
+      provider: "disabled",
+      model: config2?.model ?? "disabled",
+      dimensions: 0,
+      available: false,
+      reason: "Embeddings are disabled by configuration."
+    };
+  }
+  if (requested === "openai") {
+    const model = config2?.model ?? OPENAI_EMBEDDING_MODEL;
+    return {
+      provider: "openai",
+      model,
+      dimensions: getEmbeddingDimensions(model),
+      available: openaiAvailable,
+      reason: openaiAvailable ? void 0 : "OPENAI_API_KEY is not configured."
+    };
+  }
+  if (requested === "local") {
+    return {
+      provider: "local",
+      model: config2?.model ?? LOCAL_EMBEDDING_MODEL,
+      dimensions: LOCAL_EMBEDDING_DIMENSIONS,
+      available: true
+    };
+  }
+  if (openaiAvailable && config2?.model && config2.model.startsWith("text-embedding-")) {
+    return {
+      provider: "openai",
+      model: config2.model,
+      dimensions: getEmbeddingDimensions(config2.model),
+      available: true
+    };
+  }
+  return {
+    provider: "local",
+    model: config2?.model && !config2.model.startsWith("text-embedding-") ? config2.model : LOCAL_EMBEDDING_MODEL,
+    dimensions: LOCAL_EMBEDDING_DIMENSIONS,
+    available: true
+  };
 }
 function getClient(apiKey) {
   const key = apiKey ?? process.env.OPENAI_API_KEY;
   if (!key) {
     throw new Error(
-      "OpenAI API key not configured. Set OPENAI_API_KEY environment variable or pass apiKey option. Semantic search features are disabled. Unforgit will use FTS-only search."
+      "OpenAI API key not configured. Set OPENAI_API_KEY environment variable or use embeddings.provider=local. Unforgit can generate local embeddings without cloud credentials."
     );
   }
   if (!cachedClient || apiKey) {
@@ -247,9 +298,55 @@ function getClient(apiKey) {
   }
   return cachedClient;
 }
+function hashToUint32(value) {
+  const digest = createHash("sha256").update(value).digest();
+  return digest.readUInt32LE(0);
+}
+function normalizeToken(token) {
+  return token.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").toLowerCase();
+}
+function featuresForText(text) {
+  const words = text.split(/[^\p{L}\p{N}_-]+/u).map(normalizeToken).filter((token) => token.length > 1);
+  const features = [];
+  for (const word of words) {
+    features.push(`w:${word}`);
+    if (word.length > 4) {
+      for (let i = 0; i <= word.length - 3; i++) {
+        features.push(`g:${word.slice(i, i + 3)}`);
+      }
+    }
+  }
+  for (let i = 0; i < words.length - 1; i++) {
+    features.push(`b:${words[i]} ${words[i + 1]}`);
+  }
+  return features.length > 0 ? features : ["empty"];
+}
+function generateLocalEmbedding(text, model = LOCAL_EMBEDDING_MODEL) {
+  const vector = new Array(LOCAL_EMBEDDING_DIMENSIONS).fill(0);
+  for (const feature of featuresForText(text.trim().slice(0, 8e3))) {
+    const bucket = hashToUint32(`${model}:bucket:${feature}`) % LOCAL_EMBEDDING_DIMENSIONS;
+    const sign = hashToUint32(`${model}:sign:${feature}`) % 2 === 0 ? 1 : -1;
+    vector[bucket] += sign;
+  }
+  const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
+  const embedding = magnitude > 0 ? vector.map((value) => value / magnitude) : vector;
+  return {
+    embedding,
+    model,
+    provider: "local",
+    tokensUsed: 0
+  };
+}
 async function generateEmbedding(text, config2) {
+  const resolved = resolveEmbeddingProvider(config2);
+  if (resolved.provider === "disabled") {
+    throw new Error("Embeddings are disabled by configuration.");
+  }
+  if (resolved.provider === "local") {
+    return generateLocalEmbedding(text, resolved.model);
+  }
   const client = getClient(config2?.apiKey);
-  const model = config2?.model ?? EMBEDDING_MODEL;
+  const model = resolved.model;
   const cleanText = text.trim().slice(0, 8e3);
   const response = await client.embeddings.create({
     model,
@@ -262,6 +359,7 @@ async function generateEmbedding(text, config2) {
   return {
     embedding: data.embedding,
     model,
+    provider: "openai",
     tokensUsed: response.usage?.total_tokens ?? 0
   };
 }
@@ -298,6 +396,15 @@ function deserializeEmbedding(buffer) {
   }
   return embedding;
 }
+var EMBEDDING_DIMENSIONS_MAP = {
+  [LOCAL_EMBEDDING_MODEL]: LOCAL_EMBEDDING_DIMENSIONS,
+  "text-embedding-3-small": 1536,
+  "text-embedding-3-large": 3072,
+  "text-embedding-ada-002": 1536
+};
+function getEmbeddingDimensions(model) {
+  return EMBEDDING_DIMENSIONS_MAP[model] ?? (model.startsWith("local-") ? LOCAL_EMBEDDING_DIMENSIONS : OPENAI_EMBEDDING_DIMENSIONS);
+}
 var CONSOLIDATION_PROMPT = `You are consolidating multiple related memories into a single unified memory.
 
 Source memories:
@@ -1373,6 +1480,7 @@ var syncConfigSchema = z.object({
 });
 var embeddingConfigSchema = z.object({
   enabled: z.boolean(),
+  provider: z.enum(["auto", "local", "openai", "disabled"]).optional(),
   model: z.string(),
   autoGenerate: z.boolean()
 });
@@ -1625,7 +1733,8 @@ function defaultConfig() {
     },
     embeddings: {
       enabled: true,
-      model: "text-embedding-3-small",
+      provider: "auto",
+      model: "local-hash-multilingual-v1",
       autoGenerate: true
     },
     lifecycle: resolveLifecycleConfig()
@@ -1643,6 +1752,7 @@ var RemoteClient = class {
     this.apiKey = apiKey || process.env.UNFORGIT_API_KEY;
     this.timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   }
+  baseUrl;
   apiKey;
   timeoutMs;
   getHeaders() {
@@ -1998,6 +2108,8 @@ CREATE TABLE IF NOT EXISTS memory_embeddings (
   memory_id TEXT PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE,
   embedding BLOB NOT NULL,
   model TEXT NOT NULL,
+  provider TEXT,
+  dimensions INTEGER,
   created_at TEXT NOT NULL DEFAULT (datetime('now'))
 );
 
@@ -2184,10 +2296,21 @@ var LocalStore = class {
           memory_id TEXT PRIMARY KEY REFERENCES memories(id) ON DELETE CASCADE,
           embedding BLOB NOT NULL,
           model TEXT NOT NULL,
+          provider TEXT,
+          dimensions INTEGER,
           created_at TEXT NOT NULL DEFAULT (datetime('now'))
         );
       `);
     }
+    const embeddingColumns = this.db.prepare("PRAGMA table_info(memory_embeddings)").all();
+    const embeddingColumnNames = embeddingColumns.map((c) => c.name);
+    if (!embeddingColumnNames.includes("provider")) {
+      this.db.exec("ALTER TABLE memory_embeddings ADD COLUMN provider TEXT");
+    }
+    if (!embeddingColumnNames.includes("dimensions")) {
+      this.db.exec("ALTER TABLE memory_embeddings ADD COLUMN dimensions INTEGER");
+      this.db.prepare("UPDATE memory_embeddings SET dimensions = length(embedding) / 4 WHERE dimensions IS NULL").run();
+    }
     const usageTables = this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memory_usage'").all();
     if (usageTables.length === 0) {
       this.db.exec(`
@@ -3274,18 +3397,19 @@ var LocalStore = class {
     ).run();
     return result.changes;
   }
-  async storeEmbedding(memoryId, embedding, model) {
+  async storeEmbedding(memoryId, embedding, model, provider) {
     const now = (/* @__PURE__ */ new Date()).toISOString();
     const blob = serializeEmbedding(embedding);
+    const dimensions = embedding.length || getEmbeddingDimensions(model);
     this.db.prepare(
-      `INSERT OR REPLACE INTO memory_embeddings (memory_id, embedding, model, created_at)
-         VALUES (?, ?, ?, ?)`
-    ).run(memoryId, blob, model, now);
+      `INSERT OR REPLACE INTO memory_embeddings (memory_id, embedding, model, provider, dimensions, created_at)
+         VALUES (?, ?, ?, ?, ?, ?)`
+    ).run(memoryId, blob, model, provider ?? null, dimensions, now);
   }
   async generateAndStoreEmbedding(memoryId, text, config2) {
     try {
       const result = await generateEmbedding(text, config2);
-      await this.storeEmbedding(memoryId, result.embedding, result.model);
+      await this.storeEmbedding(memoryId, result.embedding, result.model, result.provider);
     } catch (error) {
       console.error(`Failed to generate embedding for ${memoryId}:`, error);
     }
@@ -3299,23 +3423,40 @@ var LocalStore = class {
     const row = this.db.prepare("SELECT 1 FROM memory_embeddings WHERE memory_id = ?").get(memoryId);
     return !!row;
   }
-  getAllEmbeddings(orgId, repoId) {
+  getAllEmbeddings(orgId, repoId, dimensions) {
+    const dimensionClause = dimensions ? " AND e.dimensions = ?" : "";
+    const params = dimensions ? [orgId, repoId, dimensions] : [orgId, repoId];
     const rows = this.db.prepare(
       `SELECT e.memory_id, e.embedding FROM memory_embeddings e
          JOIN memories m ON e.memory_id = m.id
-         WHERE m.org_id = ? AND m.repo_id = ? AND m.status = 'active'`
-    ).all(orgId, repoId);
+         WHERE m.org_id = ? AND m.repo_id = ? AND m.status = 'active'${dimensionClause}`
+    ).all(...params);
     return rows.map((row) => ({
       memoryId: row.memory_id,
       embedding: deserializeEmbedding(row.embedding)
     }));
   }
-  getMemoriesWithoutEmbeddings(orgId, repoId) {
+  getMemoriesWithoutEmbeddings(orgId, repoId, expected) {
+    const compatibilityClauses = ["e.memory_id IS NULL"];
+    const params = [orgId, repoId];
+    if (expected?.model) {
+      compatibilityClauses.push("e.model != ?");
+      params.push(expected.model);
+    }
+    if (expected?.provider) {
+      compatibilityClauses.push("e.provider IS NULL OR e.provider != ?");
+      params.push(expected.provider);
+    }
+    if (expected?.dimensions) {
+      compatibilityClauses.push("e.dimensions IS NULL OR e.dimensions != ?");
+      params.push(expected.dimensions);
+    }
     const rows = this.db.prepare(
       `SELECT m.* FROM memories m
          LEFT JOIN memory_embeddings e ON m.id = e.memory_id
-         WHERE m.org_id = ? AND m.repo_id = ? AND m.status = 'active' AND e.memory_id IS NULL`
-    ).all(orgId, repoId);
+         WHERE m.org_id = ? AND m.repo_id = ? AND m.status = 'active'
+           AND (${compatibilityClauses.map((clause) => `(${clause})`).join(" OR ")})`
+    ).all(...params);
     return rows.map(rowToMemory);
   }
   async recallWithEmbeddings(query, queryEmbedding) {
@@ -3323,7 +3464,7 @@ var LocalStore = class {
     if (!queryEmbedding) {
       return ftsResults;
     }
-    const allEmbeddings = this.getAllEmbeddings(query.orgId, query.repoId);
+    const allEmbeddings = this.getAllEmbeddings(query.orgId, query.repoId, queryEmbedding.length);
     if (allEmbeddings.length === 0) {
       return ftsResults;
     }
@@ -3616,6 +3757,8 @@ export {
   LifecycleScheduler,
   mergeAndRank,
   resolveVisibility,
+  isOpenAIConfigured,
+  resolveEmbeddingProvider,
   generateEmbedding,
   computeRepositoryHealth,
   generateSuggestions,
@@ -3649,4 +3792,4 @@ export {
   RemoteClient,
   LocalStore
 };
-//# sourceMappingURL=chunk-CKUDYQYP.js.map
+//# sourceMappingURL=chunk-TGDR7Y7T.js.map