squish-memory 0.8.1 → 0.9.1

package/dist/core/consolidation.js

@@ -1,40 +1,248 @@
-import { config } from '../config.js';
-async function clusterSimilarMemories(memories) {
-    const clusters = [];
-    const visited = new Set();
-    const threshold = config.consolidationSimilarityThreshold || 0.8;
-    for (const memory of memories) {
-        if (visited.has(memory.id))
+// Memory Consolidation & Deduplication
+import { eq, inArray, and } from 'drizzle-orm';
+import { getDb } from '../db/index.js';
+import { getSchema } from '../db/schema.js';
+import { createAssociation } from './associations.js';
+import { logger } from './logger.js';
+import { consolidateMemories } from './memory/consolidation.js';
+/**
+ * Run automated deduplication job
+ * Finds and marks duplicates for review or auto-merges high-confidence duplicates
+ */
+export async function runDeduplicationJob(projectId) {
+    const result = {
+        duplicatesFound: 0,
+        mergedCount: 0,
+        tokensRecovered: 0,
+        groups: [],
+    };
+    try {
+        const db = await getDb();
+        const schema = await getSchema();
+        // Get active memories for deduplication check
+        const whereClause = projectId
+            ? and(eq(schema.memories.projectId, projectId), eq(schema.memories.status, 'active'))
+            : eq(schema.memories.status, 'active');
+        const memories = await db
+            .select()
+            .from(schema.memories)
+            .where(whereClause)
+            .limit(500);
+        if (memories.length < 2) {
+            return result;
+        }
+        // Find duplicate groups using SimHash
+        const duplicateGroups = await findDuplicatesBySimHash(memories);
+        result.duplicatesFound = duplicateGroups.reduce((sum, g) => sum + g.duplicateIds.length, 0);
+        // Auto-merge high-confidence duplicates (>0.95 similarity)
+        for (const group of duplicateGroups) {
+            if (group.similarity >= 0.95) {
+                const tokensSaved = await autoMergeDuplicates(group.canonicalId, group.duplicateIds);
+                result.mergedCount += group.duplicateIds.length;
+                result.tokensRecovered += tokensSaved;
+            }
+            else {
+                // Lower confidence - just create association for review
+                for (const dupId of group.duplicateIds) {
+                    await createAssociation(group.canonicalId, dupId, 'duplicate', group.similarity);
+                }
+            }
+            result.groups.push(group);
+        }
+        logger.info('Deduplication job completed', {
+            duplicatesFound: result.duplicatesFound,
+            mergedCount: result.mergedCount,
+            tokensRecovered: result.tokensRecovered,
+        });
+    }
+    catch (error) {
+        logger.error('Deduplication job error', error);
+    }
+    return result;
+}
+/**
+ * Find duplicates using SimHash (efficient near-duplicate detection)
+ */
+async function findDuplicatesBySimHash(memories) {
+    const groups = [];
+    const processed = new Set();
+    // Compute SimHash for each memory
+    const hashes = memories.map(m => ({
+        id: m.id,
+        hash: computeSimHash(m.content),
+        content: m.content,
+        createdAt: m.createdAt,
+    }));
+    // Compare hashes to find similar content
+    for (let i = 0; i < hashes.length; i++) {
+        if (processed.has(hashes[i].id))
             continue;
-        const cluster = [memory];
-        visited.add(memory.id);
-        for (const other of memories) {
-            if (visited.has(other.id))
+        const duplicates = [];
+        let maxSimilarity = 0;
+        for (let j = i + 1; j < hashes.length; j++) {
+            if (processed.has(hashes[j].id))
                 continue;
-            const similarity = computeJaccardSimilarity(memory.content, other.content);
-            if (similarity > threshold) {
-                cluster.push(other);
-                visited.add(other.id);
+            const hammingDistance = computeHammingDistance(hashes[i].hash, hashes[j].hash);
+            const similarity = 1 - (hammingDistance / 64); // 64-bit hash
+            if (similarity >= 0.85) {
+                duplicates.push(hashes[j].id);
+                maxSimilarity = Math.max(maxSimilarity, similarity);
+                processed.add(hashes[j].id);
             }
         }
-        if (cluster.length > 0) {
-            clusters.push(cluster);
+        if (duplicates.length > 0) {
+            processed.add(hashes[i].id);
+            groups.push({
+                canonicalId: hashes[i].id, // Keep oldest as canonical
+                duplicateIds: duplicates,
+                similarity: maxSimilarity,
+                reason: 'content-similarity',
+            });
         }
     }
-    return clusters;
+    return groups;
 }
-function computeJaccardSimilarity(a, b) {
-    const tokensA = new Set(a.toLowerCase().split(/\s+/).filter(t => t.length > 2));
-    const tokensB = new Set(b.toLowerCase().split(/\s+/).filter(t => t.length > 2));
-    const intersection = new Set([...tokensA].filter(x => tokensB.has(x)));
-    const union = new Set([...tokensA, ...tokensB]);
-    return union.size > 0 ? intersection.size / union.size : 0;
+/**
+ * Compute SimHash for text (64-bit fingerprint)
+ */
+function computeSimHash(text) {
+    const tokens = text.toLowerCase().split(/\s+/).filter(t => t.length > 2);
+    const weights = new Array(64).fill(0);
+    for (const token of tokens) {
+        const hash = simpleHash(token);
+        for (let i = 0; i < 64; i++) {
+            if ((hash >> BigInt(i)) & 1n) {
+                weights[i] += 1;
+            }
+            else {
+                weights[i] -= 1;
+            }
+        }
+    }
+    let simHash = 0n;
+    for (let i = 0; i < 64; i++) {
+        if (weights[i] > 0) {
+            simHash |= (1n << BigInt(i));
+        }
+    }
+    return simHash;
 }
-function estimateTokenRecovery(cluster) {
-    let totalTokens = 0;
-    for (const mem of cluster) {
-        totalTokens += Math.ceil((mem.content || '').length / 4);
+/**
+ * Simple hash function for strings
+ */
+function simpleHash(str) {
+    let hash = 0n;
+    for (let i = 0; i < str.length; i++) {
+        const char = BigInt(str.charCodeAt(i));
+        hash = ((hash << 5n) - hash) + char;
+        hash = hash & hash; // Convert to 64bit integer
+    }
+    return hash;
+}
+/**
+ * Compute Hamming distance between two hashes
+ */
+function computeHammingDistance(a, b) {
+    let xor = a ^ b;
+    let distance = 0;
+    while (xor !== 0n) {
+        distance += Number(xor & 1n);
+        xor >>= 1n;
+    }
+    return distance;
+}
+/**
+ * Auto-merge duplicates into canonical memory
+ */
+async function autoMergeDuplicates(canonicalId, duplicateIds) {
+    try {
+        const db = await getDb();
+        const schema = await getSchema();
+        const now = new Date();
+        // Calculate tokens recovered
+        const duplicates = await db
+            .select()
+            .from(schema.memories)
+            .where(inArray(schema.memories.id, duplicateIds));
+        let tokensRecovered = 0;
+        for (const dup of duplicates) {
+            tokensRecovered += Math.ceil((dup.content?.length || 0) / 4);
+        }
+        // Mark duplicates as merged
+        await db
+            .update(schema.memories)
+            .set({
+            status: 'merged',
+            mergedInto: canonicalId,
+            mergedAt: now,
+            updatedAt: now,
+        })
+            .where(inArray(schema.memories.id, duplicateIds));
+        // Create associations for traceability
+        for (const dupId of duplicateIds) {
+            await createAssociation(canonicalId, dupId, 'merged', 0.95);
+        }
+        return tokensRecovered;
+    }
+    catch (error) {
+        logger.error('Error auto-merging duplicates', error);
+        return 0;
+    }
+}
+/**
+ * Run full consolidation job (dedup + memory consolidation)
+ */
+export async function runFullConsolidationJob(projectId) {
+    const stats = {
+        clustered: 0,
+        merged: 0,
+        tokensRecovered: 0,
+        deduped: 0,
+        consolidated: 0,
+    };
+    // Run deduplication first
+    const dedupResult = await runDeduplicationJob(projectId);
+    stats.deduped = dedupResult.duplicatesFound;
+    stats.merged = dedupResult.mergedCount;
+    stats.tokensRecovered += dedupResult.tokensRecovered;
+    // Run memory consolidation for each project
+    if (projectId) {
+        const consolidationResults = await consolidateMemories({
+            projectId,
+            minAge: 60,
+            maxImportance: 25,
+            minClusterSize: 3,
+            limit: 100,
+        });
+        stats.consolidated = consolidationResults.length;
+        stats.clustered = consolidationResults.reduce((sum, r) => sum + r.clusterSize, 0);
+    }
+    logger.info('Full consolidation job completed', stats);
+    return stats;
+}
+/**
+ * Get deduplication statistics
+ */
+export async function getDeduplicationStats(projectId) {
+    try {
+        const db = await getDb();
+        const schema = await getSchema();
+        const whereClause = projectId
+            ? eq(schema.memories.projectId, projectId)
+            : undefined;
+        const memories = await db
+            .select()
+            .from(schema.memories)
+            .where(whereClause);
+        return {
+            totalMemories: memories.length,
+            mergedMemories: memories.filter((m) => m.status === 'merged').length,
+            pendingDuplicates: 0, // Would need to query associations
+        };
+    }
+    catch (error) {
+        logger.error('Error getting deduplication stats', error);
+        return { totalMemories: 0, mergedMemories: 0, pendingDuplicates: 0 };
     }
-    return Math.max(0, totalTokens - Math.ceil(cluster[0].content.length / 4));
 }
 //# sourceMappingURL=consolidation.js.map

package/dist/core/consolidation.js.map

@@ -1 +1 @@
-{"version":3,"file":"consolidation.js","sourceRoot":"","sources":["../../core/consolidation.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAWtC,KAAK,UAAU,sBAAsB,CAAC,QAAe;IACnD,MAAM,QAAQ,GAAY,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,SAAS,GAAG,MAAM,CAAC,gCAAgC,IAAI,GAAG,CAAC;IAEjE,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAAE,SAAS;QAErC,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAEvB,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAAE,SAAS;YAEpC,MAAM,UAAU,GAAG,wBAAwB,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3E,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;gBAC3B,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,wBAAwB,CAAC,CAAS,EAAE,CAAS;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAEhF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;IAEhD,OAAO,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAc;IAC3C,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,WAAW,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC"}
+{"version":3,"file":"consolidation.js","sourceRoot":"","sources":["../../core/consolidation.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,EAAM,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAyB,MAAM,2BAA2B,CAAC;AAwBvF;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,SAAkB;IAC1D,MAAM,MAAM,GAAwB;QAClC,eAAe,EAAE,CAAC;QAClB,WAAW,EAAE,CAAC;QACd,eAAe,EAAE,CAAC;QAClB,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,8CAA8C;QAC9C,MAAM,WAAW,GAAG,SAAS;YAC3B,CAAC,CAAC,GAAG,CACD,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC,EACxC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CACrC;YACH,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEzC,MAAM,QAAQ,GAAG,MAAO,EAAU;aAC/B,MAAM,EAAE;aACR,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;aACrB,KAAK,CAAC,WAAW,CAAC;aAClB,KAAK,CAAC,GAAG,CAAC,CAAC;QAEd,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,sCAAsC;QACtC,MAAM,eAAe,GAAG,MAAM,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QAEhE,MAAM,CAAC,eAAe,GAAG,eAAe,CAAC,MAAM,CAC7C,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,EACvC,CAAC,CACF,CAAC;QAEF,2DAA2D;QAC3D,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;gBAC7B,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,YAAY,CACnB,CAAC;gBACF,MAAM,CAAC,WAAW,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC;gBAChD,MAAM,CAAC,eAAe,IAAI,WAAW,CAAC;YACxC,CAAC;iBAAM,CAAC;gBACN,wDAAwD;gBACxD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;oBACvC,MAAM,iBAAiB,CACrB,KAAK,CAAC,WAAW,EACjB,KAAK,EACL,WAAW,EACX,KAAK,CAAC,UAAU,CACjB,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE;YACzC,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;SACxC,CAAC,CAAC;IAEL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,uBAAuB,CAAC,QAAe;IACpD,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,kCAAkC;IAClC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChC,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC;QAC/B,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,SAAS,EAAE,CAAC,CAAC,SAAS;KACvB,CAAC,CAAC,CAAC;IAEJ,yCAAyC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAAE,SAAS;QAE1C,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAAE,SAAS;YAE1C,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC/E,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC,CAAC,cAAc;YAE7D,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;gBACvB,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9B,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;gBACpD,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC;gBACV,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,2BAA2B;gBACtD,YAAY,EAAE,UAAU;gBACxB,UAAU,EAAE,aAAa;gBACzB,MAAM,EAAE,oBAAoB;aAC7B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzE,MAAM,OAAO,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEtC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC7B,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QACpC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,2BAA2B;IACjD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAAC,CAAS,EAAE,CAAS;IAClD,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,OAAO,GAAG,KAAK,EAAE,EAAE,CAAC;QAClB,QAAQ,IAAI,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QAC7B,GAAG,KAAK,EAAE,CAAC;IACb,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,mBAAmB,CAChC,WAAmB,EACnB,YAAsB;IAEtB,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QAEvB,6BAA6B;QAC7B,MAAM,UAAU,GAAG,MAAO,EAAU;aACjC,MAAM,EAAE;aACR,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;aACrB,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;QAEpD,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,eAAe,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,4BAA4B;QAC5B,MAAO,EAAU;aACd,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC;aACvB,GAAG,CAAC;YACH,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,WAAW;YACvB,QAAQ,EAAE,GAAG;YACb,SAAS,EAAE,GAAG;SACf,CAAC;aACD,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;QAEpD,uCAAuC;QACvC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,iBAAiB,CAAC,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,eAAe,CAAC;IAEzB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;QACrD,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,SAAkB;IAC9D,MAAM,KAAK,GAAuB;QAChC,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;QACT,eAAe,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;QACV,YAAY,EAAE,CAAC;KAChB,CAAC;IAEF,0BAA0B;IAC1B,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC;IACzD,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC,eAAe,CAAC;IAC5C,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC;IACvC,KAAK,CAAC,eAAe,IAAI,WAAW,CAAC,eAAe,CAAC;IAErD,4CAA4C;IAC5C,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,oBAAoB,GAAG,MAAM,mBAAmB,CAAC;YACrD,SAAS;YACT,MAAM,EAAE,EAAE;YACV,aAAa,EAAE,EAAE;YACjB,cAAc,EAAE,CAAC;YACjB,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;QACH,KAAK,CAAC,YAAY,GAAG,oBAAoB,CAAC,MAAM,CAAC;QACjD,KAAK,CAAC,SAAS,GAAG,oBAAoB,CAAC,MAAM,CAC3C,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,WAAW,EAC/B,CAAC,CACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;IAEvD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,SAAkB;IAK5D,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;QAEjC,MAAM,WAAW,GAAG,SAAS;YAC3B,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;YAC1C,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,QAAQ,GAAG,MAAO,EAAU;aAC/B,MAAM,EAAE;aACR,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;aACrB,KAAK,CAAC,WAAW,CAAC,CAAC;QAEtB,OAAO;YACL,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;YACzE,iBAAiB,EAAE,CAAC,EAAE,mCAAmC;SAC1D,CAAC;IAEJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;QACzD,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAC;IACvE,CAAC;AACH,CAAC"}

package/dist/core/embeddings/google-multimodal.d.ts

@@ -0,0 +1,14 @@
+export interface MultimodalInput {
+    text?: string;
+    image?: Buffer | string;
+    video?: string;
+}
+export interface GoogleMultimodalResponse {
+    embedding: number[];
+    textEmbedding?: number[];
+    imageEmbedding?: number[];
+    videoEmbedding?: number[];
+}
+export declare function getGoogleMultimodalEmbedding(input: MultimodalInput): Promise<GoogleMultimodalResponse | null>;
+export declare function isMultimodalInput(input: any): input is MultimodalInput;
+//# sourceMappingURL=google-multimodal.d.ts.map

package/dist/core/embeddings/google-multimodal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"google-multimodal.d.ts","sourceRoot":"","sources":["../../../core/embeddings/google-multimodal.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,wBAAsB,4BAA4B,CAChD,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,wBAAwB,GAAG,IAAI,CAAC,CAqF1C;AAmFD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,GAAG,GAAG,KAAK,IAAI,eAAe,CAQtE"}

package/dist/core/embeddings/google-multimodal.js

@@ -0,0 +1,142 @@
+import { logger } from '../logger.js';
+import { config } from '../../config.js';
+export async function getGoogleMultimodalEmbedding(input) {
+    if (!config.googleCloudApiKey && !config.googleCloudProject) {
+        logger.debug('Google Cloud credentials not configured');
+        return null;
+    }
+    try {
+        const endpoint = config.googleCloudLocation === 'global'
+            ? `https://aiplatform.googleapis.com/v1/projects/${config.googleCloudProject}/locations/${config.googleCloudLocation}/publishers/google/models/multimodalembedding@001:predict`
+            : `https://${config.googleCloudLocation}-aiplatform.googleapis.com/v1/projects/${config.googleCloudProject}/locations/${config.googleCloudLocation}/publishers/google/models/multimodalembedding@001:predict`;
+        const instances = [];
+        if (input.text) {
+            instances.push({ text: input.text });
+        }
+        if (input.image) {
+            const imageBytes = typeof input.image === 'string'
+                ? input.image
+                : input.image.toString('base64');
+            instances.push({
+                image: {
+                    bytesBase64Encoded: imageBytes,
+                },
+            });
+        }
+        if (input.video) {
+            instances.push({
+                video: {
+                    gcsUri: input.video,
+                },
+            });
+        }
+        if (instances.length === 0) {
+            return null;
+        }
+        const response = await fetch(endpoint, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${await getGoogleAccessToken()}`,
+            },
+            body: JSON.stringify({
+                instances,
+                parameters: {
+                    dimension: 1408,
+                },
+            }),
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            logger.warn(`Google Multimodal embeddings failed: ${response.status} ${errorText}`);
+            return null;
+        }
+        const data = await response.json();
+        const predictions = data.predictions || [];
+        const result = {
+            embedding: predictions[0]?.embedding || [],
+        };
+        if (input.text && predictions[0]?.textEmbedding) {
+            result.textEmbedding = predictions[0].textEmbedding;
+        }
+        if (input.image && predictions[0]?.imageEmbedding) {
+            result.imageEmbedding = predictions[0].imageEmbedding;
+        }
+        if (input.video && predictions[0]?.videoEmbedding) {
+            result.videoEmbedding = predictions[0].videoEmbedding;
+        }
+        return result.embedding.length > 0 ? result : null;
+    }
+    catch (error) {
+        logger.error('Google Multimodal embedding error:', error);
+        return null;
+    }
+}
+let cachedAccessToken = null;
+let tokenExpiry = 0;
+async function getGoogleAccessToken() {
+    if (config.googleCloudApiKey) {
+        return config.googleCloudApiKey;
+    }
+    if (cachedAccessToken && Date.now() < tokenExpiry) {
+        return cachedAccessToken;
+    }
+    if (!process.env.GOOGLE_APPLICATION_CREDENTIALS) {
+        throw new Error('Google Application Credentials not set');
+    }
+    try {
+        const credentials = JSON.parse(await import('fs').then(fs => fs.promises.readFile(process.env.GOOGLE_APPLICATION_CREDENTIALS, 'utf-8')));
+        const now = Math.floor(Date.now() / 1000);
+        const jwt = await createJWT(credentials);
+        const tokenResponse = await fetch('https://oauth2.googleapis.com/token', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+            body: new URLSearchParams({
+                grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+                assertion: jwt,
+            }),
+        });
+        if (!tokenResponse.ok) {
+            throw new Error('Failed to get Google access token');
+        }
+        const tokenData = await tokenResponse.json();
+        cachedAccessToken = tokenData.access_token;
+        tokenExpiry = now + tokenData.expires_in - 60;
+        return cachedAccessToken;
+    }
+    catch (error) {
+        logger.error('Failed to get Google access token:', error);
+        throw error;
+    }
+}
+async function createJWT(credentials) {
+    const now = Math.floor(Date.now() / 1000);
+    const header = {
+        alg: 'RS256',
+        typ: 'JWT',
+    };
+    const payload = {
+        iss: credentials.client_email,
+        sub: credentials.client_email,
+        aud: 'https://oauth2.googleapis.com/token',
+        iat: now,
+        exp: now + 3600,
+        scope: 'https://www.googleapis.com/auth/cloud-platform',
+    };
+    const crypto = await import('crypto');
+    const encodedHeader = Buffer.from(JSON.stringify(header)).toString('base64url');
+    const encodedPayload = Buffer.from(JSON.stringify(payload)).toString('base64url');
+    const signatureInput = `${encodedHeader}.${encodedPayload}`;
+    const sign = crypto.createSign('RSA-SHA256');
+    sign.update(signatureInput);
+    const signature = sign.sign(credentials.private_key, 'base64url');
+    return `${signatureInput}.${signature}`;
+}
+export function isMultimodalInput(input) {
+    return (typeof input === 'object' &&
+        (typeof input.text === 'string' ||
+            Buffer.isBuffer(input.image) ||
+            typeof input.image === 'string' ||
+            typeof input.video === 'string'));
+}
+//# sourceMappingURL=google-multimodal.js.map

package/dist/core/embeddings/google-multimodal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"google-multimodal.js","sourceRoot":"","sources":["../../../core/embeddings/google-multimodal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAezC,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,KAAsB;IAEtB,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC5D,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,CAAC,mBAAmB,KAAK,QAAQ;YACtD,CAAC,CAAC,iDAAiD,MAAM,CAAC,kBAAkB,cAAc,MAAM,CAAC,mBAAmB,2DAA2D;YAC/K,CAAC,CAAC,WAAW,MAAM,CAAC,mBAAmB,0CAA0C,MAAM,CAAC,kBAAkB,cAAc,MAAM,CAAC,mBAAmB,2DAA2D,CAAC;QAEhN,MAAM,SAAS,GAAU,EAAE,CAAC;QAE5B,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,UAAU,GAAG,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;gBAChD,CAAC,CAAC,KAAK,CAAC,KAAK;gBACb,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEnC,SAAS,CAAC,IAAI,CAAC;gBACb,KAAK,EAAE;oBACL,kBAAkB,EAAE,UAAU;iBAC/B;aACF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,SAAS,CAAC,IAAI,CAAC;gBACb,KAAK,EAAE;oBACL,MAAM,EAAE,KAAK,CAAC,KAAK;iBACpB;aACF,CAAC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YACrC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,MAAM,oBAAoB,EAAE,EAAE;aAC1D;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,SAAS;gBACT,UAAU,EAAE;oBACV,SAAS,EAAE,IAAI;iBAChB;aACF,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,wCAAwC,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA6B,CAAC;QAC9D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;QAE3C,MAAM,MAAM,GAA6B;YACvC,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,EAAE;SAC3C,CAAC;QAEF,IAAI,KAAK,CAAC,IAAI,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC;YAChD,MAAM,CAAC,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;QACtD,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;YAClD,MAAM,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;QACxD,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;YAClD,MAAM,CAAC,cAAc,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;QACxD,CAAC;QAED,OAAO,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;QAC1D,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,IAAI,iBAAiB,GAAkB,IAAI,CAAC;AAC5C,IAAI,WAAW,GAAW,CAAC,CAAC;AAE5B,KAAK,UAAU,oBAAoB;IACjC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC,iBAAiB,CAAC;IAClC,CAAC;IAED,IAAI,iBAAiB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,EAAE,CAAC;QAClD,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,8BAA8B,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAC5B,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAC3B,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,8BAA+B,EAAE,OAAO,CAAC,CAC3E,CACF,CAAC;QAEF,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,WAAW,CAAC,CAAC;QAEzC,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,qCAAqC,EAAE;YACvE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,mCAAmC,EAAE;YAChE,IAAI,EAAE,IAAI,eAAe,CAAC;gBACxB,UAAU,EAAE,6CAA6C;gBACzD,SAAS,EAAE,GAAG;aACf,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,EAAkD,CAAC;QAC7F,iBAAiB,GAAG,SAAS,CAAC,YAAY,CAAC;QAC3C,WAAW,GAAG,GAAG,GAAG,SAAS,CAAC,UAAU,GAAG,EAAE,CAAC;QAE9C,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;QAC1D,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,WAAgB;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IAE1C,MAAM,MAAM,GAAG;QACb,GAAG,EAAE,OAAO;QACZ,GAAG,EAAE,KAAK;KACX,CAAC;IAEF,MAAM,OAAO,GAAG;QACd,GAAG,EAAE,WAAW,CAAC,YAAY;QAC7B,GAAG,EAAE,WAAW,CAAC,YAAY;QAC7B,GAAG,EAAE,qCAAqC;QAC1C,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG,GAAG,IAAI;QACf,KAAK,EAAE,gDAAgD;KACxD,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEtC,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAElF,MAAM,cAAc,GAAG,GAAG,aAAa,IAAI,cAAc,EAAE,CAAC;IAE5D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IAElE,OAAO,GAAG,cAAc,IAAI,SAAS,EAAE,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAU;IAC1C,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;YAC9B,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;YAC5B,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;YAC/B,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAClC,CAAC;AACJ,CAAC"}

package/dist/core/embeddings.d.ts

@@ -1,5 +1,6 @@
-export type EmbeddingProvider = 'openai' | 'ollama' | 'local' | 'none' | 'auto' | 'qmd' | 'hybrid';
-export declare function getEmbedding(input: string): Promise<number[] | null>;
+import { MultimodalInput } from './embeddings/google-multimodal.js';
+export type EmbeddingProvider = 'openai' | 'ollama' | 'local' | 'none' | 'auto' | 'qmd' | 'hybrid' | 'google-multimodal';
+export declare function getEmbedding(input: string | MultimodalInput): Promise<number[] | null>;
 /**
  * Get embeddings for multiple inputs in parallel batches
  * Processes inputs in batches to respect rate limits while parallelizing

package/dist/core/embeddings.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../core/embeddings.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,CAAC;AAiCnG,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAgE1E;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,GAAE,MAAW,GACrB,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAuCjC;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,IAAI,CAE1C;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAK1E"}
+{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../core/embeddings.ts"],"names":[],"mappings":"AAEA,OAAO,EAAmD,eAAe,EAAE,MAAM,mCAAmC,CAAC;AAGrH,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,mBAAmB,CAAC;AAiCzH,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAsF5F;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,GAAE,MAAW,GACrB,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAuCjC;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,IAAI,CAE1C;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAK1E"}

package/dist/core/embeddings.js

@@ -1,5 +1,6 @@
 import { config } from '../config.js';
 import { getQMDClient } from './embeddings/qmd-client.js';
+import { getGoogleMultimodalEmbedding, isMultimodalInput } from './embeddings/google-multimodal.js';
 import { logger } from './logger.js';
 // Simple in-memory cache for embeddings (LRU with 1000 entries)
 const embeddingCache = new Map();
@@ -29,62 +30,83 @@ function setCachedEmbedding(key, embedding) {
     embeddingCache.set(key, embedding);
 }
 export async function getEmbedding(input) {
-    if (!input || typeof input !== 'string') {
+    if (!input || (typeof input !== 'string' && !isMultimodalInput(input))) {
         return null;
     }
     const provider = config.embeddingsProvider;
-    const cacheKey = getCacheKey(input, provider);
+    const cacheKey = typeof input === 'string'
+        ? getCacheKey(input, provider)
+        : getCacheKey(JSON.stringify(input), provider);
     // Check cache first
     const cached = getCachedEmbedding(cacheKey);
     if (cached) {
         return cached;
     }
     let result = null;
-    if (provider === 'none') {
-        result = null;
-    }
-    else if (provider === 'qmd') {
-        result = await getQMDEmbedding(input);
-        // Fallback if QMD fails (unless qmd-only mode)
-        if (!result && config.qmdFallbackMode !== 'qmd-only') {
-            result = getLocalEmbedding(input);
+    // Handle multimodal input
+    if (isMultimodalInput(input) && (provider === 'google-multimodal' || provider === 'hybrid')) {
+        const multimodalResult = await getGoogleMultimodalEmbedding(input);
+        if (multimodalResult) {
+            result = multimodalResult.embedding;
         }
     }
-    else if (provider === 'hybrid') {
-        // Hybrid mode: Try QMD first, then cloud providers, then local
-        if (config.qmdEnabled) {
-            result = await getQMDEmbedding(input);
+    // Handle text-only input
+    if (!result && typeof input === 'string') {
+        const textInput = input;
+        if (provider === 'none') {
+            result = null;
         }
-        if (!result && config.qmdFallbackMode !== 'qmd-only') {
-            result = await getOllamaEmbedding(input);
+        else if (provider === 'google-multimodal') {
+            const multimodalResult = await getGoogleMultimodalEmbedding({ text: textInput });
+            result = multimodalResult?.embedding || null;
         }
-        if (!result && config.qmdFallbackMode !== 'qmd-only' && config.qmdFallbackMode !== 'local-only') {
-            result = await getOpenAiEmbedding(input);
+        else if (provider === 'qmd') {
+            result = await getQMDEmbedding(textInput);
+            // Fallback if QMD fails (unless qmd-only mode)
+            if (!result && config.qmdFallbackMode !== 'qmd-only') {
+                result = getLocalEmbedding(textInput);
+            }
         }
-        if (!result) {
-            result = getLocalEmbedding(input);
+        else if (provider === 'hybrid') {
+            // Hybrid mode: Try Google Multimodal first, then QMD, then cloud providers, then local
+            if (config.multimodalEmbeddingsEnabled) {
+                const multimodalResult = await getGoogleMultimodalEmbedding({ text: textInput });
+                result = multimodalResult?.embedding || null;
+            }
+            if (!result && config.qmdEnabled) {
+                result = await getQMDEmbedding(textInput);
+            }
+            if (!result && config.qmdFallbackMode !== 'qmd-only') {
+                result = await getOllamaEmbedding(textInput);
+            }
+            if (!result && config.qmdFallbackMode !== 'qmd-only' && config.qmdFallbackMode !== 'local-only') {
+                result = await getOpenAiEmbedding(textInput);
+            }
+            if (!result) {
+                result = getLocalEmbedding(textInput);
+            }
         }
-    }
-    else if (provider === 'openai') {
-        result = await getOpenAiEmbedding(input);
-    }
-    else if (provider === 'ollama') {
-        result = await getOllamaEmbedding(input);
-    }
-    else if (provider === 'local') {
-        result = getLocalEmbedding(input);
-    }
-    else {
-        // Auto mode: use local TF-IDF by default (fast, no API needed)
-        // Only try external providers if explicitly configured
-        if (config.openAiApiKey) {
-            result = await getOpenAiEmbedding(input);
+        else if (provider === 'openai') {
+            result = await getOpenAiEmbedding(textInput);
+        }
+        else if (provider === 'ollama') {
+            result = await getOllamaEmbedding(textInput);
         }
-        if (!result && config.ollamaUrl) {
-            result = await getOllamaEmbedding(input);
+        else if (provider === 'local') {
+            result = getLocalEmbedding(textInput);
         }
-        if (!result) {
-            result = getLocalEmbedding(input);
+        else {
+            // Auto mode: use local TF-IDF by default (fast, no API needed)
+            // Only try external providers if explicitly configured
+            if (config.openAiApiKey) {
+                result = await getOpenAiEmbedding(textInput);
+            }
+            if (!result && config.ollamaUrl) {
+                result = await getOllamaEmbedding(textInput);
+            }
+            if (!result) {
+                result = getLocalEmbedding(textInput);
+            }
         }
     }
     // Cache the result if valid