prism-mcp-server 7.8.4 → 7.8.6

package/dist/dashboard/server.js

@@ -445,6 +445,7 @@ return false;}
                             let cursorId = undefined;
                             let iterations = 0;
                             const MAX_ITERATIONS = 100; // safety cap: 100 × 50 = 5000 entries max
+                            let lastBackfillError = undefined;
                             while (hasMore && iterations < MAX_ITERATIONS) {
                                 iterations++;
                                 const result = await backfillEmbeddingsHandler({ dry_run: false, limit: 50, _cursor_id: cursorId });
@@ -452,6 +453,8 @@ return false;}
                                 if (bStats) {
                                     repairedCount += bStats.repaired;
                                     failedCount += bStats.failed;
+                                    if (bStats.error)
+                                        lastBackfillError = bStats.error;
                                     if (bStats.last_id)
                                         cursorId = bStats.last_id;
                                     else
@@ -464,8 +467,10 @@ return false;}
                                 }
                             }
                             cleanupMessages.push(`Repaired ${repairedCount} embeddings`);
-                            if (failedCount > 0)
-                                cleanupMessages.push(`Failed to repair ${failedCount} embeddings`);
+                            if (failedCount > 0) {
+                                const errMsg = lastBackfillError ? ` (${lastBackfillError})` : '';
+                                cleanupMessages.push(`Failed to repair ${failedCount} embeddings${errMsg}`);
+                            }
                         }
                         catch (err) {
                             console.error("[Dashboard] Failed to backfill embeddings:", err);

package/dist/storage/supabase.js

@@ -1442,8 +1442,43 @@ export class SupabaseStorage {
     }
     // ─── v7.5: Semantic Consolidation ────────────────────────────────
     async upsertSemanticKnowledge(data) {
-        // For now we just implement graceful degradation/no-op on Supabase until the SQL is deployed.
-        debugLog(`[SupabaseStorage] upsertSemanticKnowledge is not fully implemented in Supabase yet. Skipping for ${data.concept}.`);
-        return crypto.randomUUID();
+        const userId = data.userId || PRISM_USER_ID;
+        // Check if concept already exists
+        const existing = await supabaseGet("semantic_knowledge", {
+            project: `eq.${data.project}`,
+            concept: `eq.${data.concept}`,
+            select: "id,instances,confidence",
+            limit: "1"
+        });
+        const rows = Array.isArray(existing) ? existing : [];
+        if (rows.length > 0) {
+            const row = rows[0];
+            const newConfidence = Math.min(1.0, (row.confidence || 0) + 0.1);
+            const newInstances = (row.instances || 0) + 1;
+            await supabasePatch("semantic_knowledge", {
+                instances: newInstances,
+                confidence: newConfidence,
+                updated_at: new Date().toISOString()
+            }, {
+                id: `eq.${row.id}`
+            });
+            return row.id;
+        }
+        else {
+            const id = crypto.randomUUID();
+            await supabasePost("semantic_knowledge", {
+                id,
+                project: data.project,
+                user_id: userId,
+                concept: data.concept,
+                description: data.description,
+                confidence: 0.5,
+                instances: 1,
+                related_entities: data.related_entities ? JSON.stringify(data.related_entities) : "[]",
+                created_at: new Date().toISOString(),
+                updated_at: new Date().toISOString()
+            });
+            return id;
+        }
     }
 }

package/dist/tools/hygieneHandlers.js

@@ -98,46 +98,74 @@ export async function backfillEmbeddingsHandler(args) {
             isError: false,
         };
     }
-    // Generate embeddings for each entry
     let repaired = 0;
     let failed = 0;
-    for (const entry of entries) {
+    let lastError = undefined;
+    const validEntries = entries.map(e => {
+        const entry = e;
+        const textToEmbed = [
+            entry.summary || "",
+            ...(entry.decisions || []),
+        ].filter(Boolean).join(" | ");
+        return { entry, textToEmbed };
+    }).filter(x => {
+        if (!x.textToEmbed.trim()) {
+            debugLog(`[backfill] Skipping entry ${x.entry.id}: no text content`);
+            failed++;
+            return false;
+        }
+        return true;
+    });
+    if (validEntries.length > 0) {
+        const provider = getLLMProvider();
         try {
-            const e = entry;
-            const textToEmbed = [
-                e.summary || "",
-                ...(e.decisions || []),
-            ].filter(Boolean).join(" | ");
-            if (!textToEmbed.trim()) {
-                debugLog(`[backfill] Skipping entry ${e.id}: no text content`);
-                failed++;
-                continue;
+            let embeddings;
+            if (provider.generateEmbeddings) {
+                // Use batch API
+                embeddings = await provider.generateEmbeddings(validEntries.map(x => x.textToEmbed));
             }
-            const embedding = await getLLMProvider().generateEmbedding(textToEmbed);
-            // Build atomic patch — float32 + TurboQuant in ONE DB update
-            const patchData = {
-                embedding: JSON.stringify(embedding),
-            };
-            // TurboQuant: compress alongside repair (non-fatal)
-            try {
-                const { getDefaultCompressor, serialize } = await import("../utils/turboquant.js");
-                const compressor = getDefaultCompressor();
-                const compressed = compressor.compress(embedding);
-                const buf = serialize(compressed);
-                patchData.embedding_compressed = buf.toString("base64");
-                patchData.embedding_format = `turbo${compressor.bits}`;
-                patchData.embedding_turbo_radius = compressed.radius;
+            else {
+                // Fallback to sequential if batching is not supported by the adapter
+                embeddings = [];
+                for (const { textToEmbed } of validEntries) {
+                    embeddings.push(await provider.generateEmbedding(textToEmbed));
+                }
             }
-            catch (turboErr) {
-                debugLog(`[backfill] TurboQuant compression failed for ${e.id} (non-fatal): ${turboErr.message}`);
+            for (let i = 0; i < validEntries.length; i++) {
+                const { entry } = validEntries[i];
+                const embedding = embeddings[i];
+                try {
+                    const patchData = {
+                        embedding: JSON.stringify(embedding),
+                    };
+                    try {
+                        const { getDefaultCompressor, serialize } = await import("../utils/turboquant.js");
+                        const compressor = getDefaultCompressor();
+                        const compressed = compressor.compress(embedding);
+                        const buf = serialize(compressed);
+                        patchData.embedding_compressed = buf.toString("base64");
+                        patchData.embedding_format = `turbo${compressor.bits}`;
+                        patchData.embedding_turbo_radius = compressed.radius;
+                    }
+                    catch (turboErr) {
+                        debugLog(`[backfill] TurboQuant compression failed for ${entry.id} (non-fatal): ${turboErr.message}`);
+                    }
+                    await storage.patchLedger(entry.id, patchData);
+                    repaired++;
+                    debugLog(`[backfill] ✅ Repaired ${entry.id} (${entry.project})`);
+                }
+                catch (entryErr) {
+                    failed++;
+                    lastError = entryErr instanceof Error ? entryErr.message : String(entryErr);
+                    console.error(`[backfill] ❌ Failed ${entry.id}: ${lastError}`);
+                }
             }
-            await storage.patchLedger(e.id, patchData);
-            repaired++;
-            debugLog(`[backfill] ✅ Repaired ${e.id} (${e.project})`);
         }
         catch (err) {
-            failed++;
-            console.error(`[backfill] ❌ Failed ${entry.id}: ${err instanceof Error ? err.message : err}`);
+            // Embedding API call itself failed — entire batch is lost.
+            failed += validEntries.length;
+            lastError = err instanceof Error ? err.message : String(err);
+            console.error(`[backfill] ❌ Embedding API failed for batch of ${validEntries.length}: ${lastError}`);
         }
     }
     return {
@@ -152,7 +180,7 @@ export async function backfillEmbeddingsHandler(args) {
                         : `All entries now have embeddings for semantic search.`),
             }],
         isError: false,
-        _stats: { repaired, failed, last_id: entries[entries.length - 1]?.id },
+        _stats: { repaired, failed, error: lastError, last_id: entries[entries.length - 1]?.id },
     };
 }
 export async function sessionBackfillLinksHandler(args) {

package/dist/utils/llm/adapters/traced.js

@@ -46,6 +46,10 @@ import { getTracer } from "../../telemetry.js";
 export class TracingLLMProvider {
     inner;
     providerName;
+    /**
+     * Optional batch embeddings generation support.
+     */
+    generateEmbeddings;
     /**
      * The optional VLM method is declared here as a typed property so TypeScript
      * knows about it. It is assigned (or left undefined) in the constructor body
@@ -62,6 +66,37 @@ export class TracingLLMProvider {
     constructor(inner, providerName) {
         this.inner = inner;
         this.providerName = providerName;
+        // ── Batch Embeddings: conditional own-property assignment ───────────────
+        if (inner.generateEmbeddings) {
+            const innerEmbeds = inner.generateEmbeddings.bind(inner);
+            const providerName = this.providerName;
+            this.generateEmbeddings = async (texts) => {
+                const span = getTracer().startSpan("llm.generate_embeddings_batch", {
+                    attributes: {
+                        "llm.provider": providerName,
+                        "llm.batch_size": texts.length,
+                    },
+                });
+                return context.with(trace.setSpan(context.active(), span), async () => {
+                    try {
+                        const result = await innerEmbeds(texts);
+                        span.setStatus({ code: SpanStatusCode.OK });
+                        return result;
+                    }
+                    catch (err) {
+                        span.recordException(err instanceof Error ? err : new Error(String(err)));
+                        span.setStatus({
+                            code: SpanStatusCode.ERROR,
+                            message: err instanceof Error ? err.message : String(err),
+                        });
+                        throw err;
+                    }
+                    finally {
+                        span.end();
+                    }
+                });
+            };
+        }
         // ── VLM method: conditional own-property assignment ──────────────────
         // REVIEWER NOTE: TypeScript class methods always appear on the prototype,
         // which means `if (llm.generateImageDescription)` would always be truthy

package/dist/utils/llm/adapters/voyage.js

@@ -78,53 +78,78 @@ export class VoyageAdapter {
             "Set text_provider to 'anthropic', 'openai', or 'gemini' in the dashboard.");
     }
     // ─── Embedding Generation ────────────────────────────────────────────────
-    async generateEmbedding(text) {
-        if (!text || !text.trim()) {
-            throw new Error("[VoyageAdapter] generateEmbedding called with empty text");
-        }
+    async generateEmbeddings(texts) {
+        if (!texts || texts.length === 0)
+            return [];
         // Truncate to character limit (consistent with other adapters)
-        const truncated = text.length > MAX_EMBEDDING_CHARS
+        const truncatedTexts = texts.map(text => text.length > MAX_EMBEDDING_CHARS
             ? text.slice(0, MAX_EMBEDDING_CHARS).replace(/\s+\S*$/, "")
-            : text;
+            : text);
         const model = getSettingSync("voyage_model", DEFAULT_MODEL);
-        debugLog(`[VoyageAdapter] generateEmbedding — model=${model}, chars=${truncated.length}`);
+        debugLog(`[VoyageAdapter] generateEmbeddings batch — model=${model}, count=${texts.length}`);
         const requestBody = {
-            input: [truncated],
+            input: truncatedTexts,
             model,
             // We do NOT send output_dimension here because Voyage's API explicitly
             // restricts it to [256, 512, 1024, 2048] for MRL models. We will
             // manually slice the 1024-dim result down to 768 client-side.
         };
-        const response = await fetch(`${VOYAGE_API_BASE}/embeddings`, {
-            method: "POST",
-            headers: {
-                "Authorization": `Bearer ${this.apiKey}`,
-                "Content-Type": "application/json",
-            },
-            body: JSON.stringify(requestBody),
-        });
-        if (!response.ok) {
+        let response = null;
+        let retries = 0;
+        const maxRetries = 4;
+        const baseDelayMs = 15000; // 15 seconds base delay
+        while (true) {
+            response = await fetch(`${VOYAGE_API_BASE}/embeddings`, {
+                method: "POST",
+                headers: {
+                    "Authorization": `Bearer ${this.apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify(requestBody),
+            });
+            if (response.ok) {
+                break;
+            }
             const errorText = await response.text().catch(() => "unknown error");
+            if (response.status === 429 && retries < maxRetries) {
+                // Simple backoff: baseDelayMs * (retries + 1) -> 15s, 30s, 45s, 60s
+                const delay = baseDelayMs * (retries + 1);
+                retries++;
+                debugLog(`[VoyageAdapter] Rate limited (429). Retrying in ${delay}ms... (Attempt ${retries}/${maxRetries}): ${errorText.substring(0, 50)}...`);
+                await new Promise(resolve => setTimeout(resolve, delay));
+                continue;
+            }
             throw new Error(`[VoyageAdapter] API request failed — status=${response.status}: ${errorText}`);
         }
         const data = (await response.json());
-        let embedding = data?.data?.[0]?.embedding;
-        if (!Array.isArray(embedding)) {
-            throw new Error("[VoyageAdapter] Unexpected response format — no embedding array found");
-        }
-        // Client-side MRL Truncation: 
-        // Voyage models returning 1024 dims can be safely sliced to 768 since they 
-        // are trained with Matryoshka Representation Learning.
-        if (embedding.length > EMBEDDING_DIMS) {
-            embedding = embedding.slice(0, EMBEDDING_DIMS);
-        }
-        // Dimension guard: Prism's DB schema requires exactly 768 dims.
-        if (embedding.length !== EMBEDDING_DIMS) {
-            throw new Error(`[VoyageAdapter] Embedding dimension mismatch: expected ${EMBEDDING_DIMS}, ` +
-                `got ${embedding.length}. Make sure you are using a model that returns at least 768 dims.`);
+        const embeddings = data?.data?.map(d => d.embedding) || [];
+        if (embeddings.length !== texts.length) {
+            throw new Error(`[VoyageAdapter] Unexpected response length — expected ${texts.length}, got ${embeddings.length}`);
         }
-        debugLog(`[VoyageAdapter] Embedding generated — dims=${embedding.length}, ` +
+        const processedEmbeddings = embeddings.map(emb => {
+            let embedding = emb;
+            // Client-side MRL Truncation: 
+            // Voyage models returning 1024 dims can be safely sliced to 768 since they 
+            // are trained with Matryoshka Representation Learning.
+            if (embedding.length > EMBEDDING_DIMS) {
+                embedding = embedding.slice(0, EMBEDDING_DIMS);
+            }
+            // Dimension guard: Prism's DB schema requires exactly 768 dims.
+            if (embedding.length !== EMBEDDING_DIMS) {
+                throw new Error(`[VoyageAdapter] Embedding dimension mismatch: expected ${EMBEDDING_DIMS}, ` +
+                    `got ${embedding.length}. Make sure you are using a model that returns at least 768 dims.`);
+            }
+            return embedding;
+        });
+        debugLog(`[VoyageAdapter] Batch embeddings generated — count=${processedEmbeddings.length}, ` +
             `tokens_used=${data.usage?.total_tokens ?? "unknown"}`);
-        return embedding;
+        return processedEmbeddings;
+    }
+    async generateEmbedding(text) {
+        if (!text || !text.trim()) {
+            throw new Error("[VoyageAdapter] generateEmbedding called with empty text");
+        }
+        const results = await this.generateEmbeddings([text]);
+        return results[0];
     }
 }

package/dist/utils/llm/factory.js

@@ -117,6 +117,10 @@ export function getLLMProvider() {
             generateText: textAdapter.generateText.bind(textAdapter),
             generateEmbedding: embedAdapter.generateEmbedding.bind(embedAdapter),
         };
+        // Wire batch embeddings if the embed adapter supports it (e.g. VoyageAdapter).
+        if (embedAdapter.generateEmbeddings) {
+            composed.generateEmbeddings = embedAdapter.generateEmbeddings.bind(embedAdapter);
+        }
         // Pass VLM support through from the text adapter if it exists.
         // generateImageDescription is a text-generation concern (it calls the
         // text/vision model, not the embedding model). The text adapter owns it.
@@ -141,6 +145,9 @@ export function getLLMProvider() {
             generateText: fallback.generateText.bind(fallback),
             generateEmbedding: fallback.generateEmbedding.bind(fallback),
         };
+        if (typeof fallback.generateEmbeddings === 'function') {
+            fallbackComposed.generateEmbeddings = fallback.generateEmbeddings.bind(fallback);
+        }
         if (fallback.generateImageDescription) {
             fallbackComposed.generateImageDescription = fallback.generateImageDescription.bind(fallback);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prism-mcp-server",
-  "version": "7.8.4",
+  "version": "7.8.6",
   "mcpName": "io.github.dcostenco/prism-mcp",
   "description": "The Mind Palace for AI Agents — a true Cognitive Architecture with Hebbian learning (episodic→semantic consolidation), ACT-R spreading activation (multi-hop causal reasoning), uncertainty-aware rejection gates (agents that know when they don't know), adversarial evaluation (anti-sycophancy), fail-closed Dark Factory pipelines, persistent memory (SQLite/Supabase), multi-agent Hivemind, time travel & visual dashboard. Zero-config local mode.",
   "module": "index.ts",