prism-mcp-server 5.2.0 → 5.5.0

package/dist/scholar/webScholar.js

@@ -0,0 +1,258 @@
+import { BRAVE_API_KEY, FIRECRAWL_API_KEY, PRISM_SCHOLAR_MAX_ARTICLES_PER_RUN, PRISM_USER_ID, PRISM_SCHOLAR_TOPICS, PRISM_ENABLE_HIVEMIND } from "../config.js";
+import { getStorage } from "../storage/index.js";
+import { debugLog } from "../utils/logger.js";
+import { getLLMProvider } from "../utils/llm/factory.js";
+import { randomUUID } from "node:crypto";
+import { performWebSearchRaw } from "../utils/braveApi.js";
+import { getTracer } from "../utils/telemetry.js";
+import { searchYahooFree, scrapeArticleLocal } from "./freeSearch.js";
+// ─── Hivemind Integration Helpers ────────────────────────────
+const SCHOLAR_PROJECT = "prism-scholar";
+const SCHOLAR_ROLE = "scholar";
+/**
+ * Phase 1: Register the Scholar as a Hivemind agent and emit heartbeats.
+ * Shows up on the Dashboard Radar as 🧠 with the current research topic.
+ * Gracefully no-ops when Hivemind is disabled.
+ */
+async function hivemindRegister(topic) {
+    if (!PRISM_ENABLE_HIVEMIND)
+        return;
+    try {
+        const storage = await getStorage();
+        await storage.registerAgent({
+            project: SCHOLAR_PROJECT,
+            user_id: PRISM_USER_ID,
+            role: SCHOLAR_ROLE,
+            agent_name: "Web Scholar",
+            status: "active",
+            current_task: `Researching: ${topic}`,
+        });
+        debugLog(`[WebScholar] 🐝 Registered on Hivemind Radar (topic: ${topic})`);
+    }
+    catch (err) {
+        debugLog(`[WebScholar] Hivemind registration failed (non-fatal): ${err}`);
+    }
+}
+async function hivemindHeartbeat(task) {
+    if (!PRISM_ENABLE_HIVEMIND)
+        return;
+    try {
+        const storage = await getStorage();
+        await storage.heartbeatAgent(SCHOLAR_PROJECT, PRISM_USER_ID, SCHOLAR_ROLE, task);
+    }
+    catch { /* non-fatal */ }
+}
+async function hivemindIdle() {
+    if (!PRISM_ENABLE_HIVEMIND)
+        return;
+    try {
+        const storage = await getStorage();
+        await storage.updateAgentStatus(SCHOLAR_PROJECT, PRISM_USER_ID, SCHOLAR_ROLE, "idle");
+    }
+    catch { /* non-fatal */ }
+}
+/**
+ * Phase 2: Broadcast a Telepathy alert after a successful research run.
+ * Active dev/qa agents will see "[🐝 SCHOLAR]" in their next tool response.
+ * Uses console.error to log the broadcast — the Watchdog sweep will pick up
+ * the Scholar's state change and generate alerts for active agents.
+ */
+async function hivemindBroadcast(topic, articleCount) {
+    if (!PRISM_ENABLE_HIVEMIND)
+        return;
+    try {
+        const storage = await getStorage();
+        // Update Scholar's current_task so the Watchdog and Dashboard show the result
+        await storage.heartbeatAgent(SCHOLAR_PROJECT, PRISM_USER_ID, SCHOLAR_ROLE, `✅ Completed: "${topic}" — ${articleCount} articles synthesized`);
+        // Log the broadcast — visible to operators watching the process
+        console.error(`[WebScholar] 🐝 TELEPATHY: New research on "${topic}" — ` +
+            `${articleCount} articles synthesized. Active agents will see results in knowledge search.`);
+    }
+    catch (err) {
+        debugLog(`[WebScholar] Telepathy broadcast failed (non-fatal): ${err}`);
+    }
+}
+/**
+ * Phase 3: Task-aware topic selection.
+ * If Hivemind is active, check what other agents are working on and
+ * bias toward configured topics that overlap with their active tasks.
+ * Falls back to random selection if no matches or Hivemind is off.
+ */
+async function selectTopic() {
+    const topics = PRISM_SCHOLAR_TOPICS;
+    if (!topics || topics.length === 0)
+        return "";
+    // Default: random pick
+    const randomPick = topics[Math.floor(Math.random() * topics.length)];
+    if (!PRISM_ENABLE_HIVEMIND)
+        return randomPick;
+    try {
+        const storage = await getStorage();
+        const allAgents = await storage.getAllAgents(PRISM_USER_ID);
+        const activeTasks = allAgents
+            .filter(a => a.role !== SCHOLAR_ROLE && a.status === "active" && a.current_task)
+            .map(a => a.current_task.toLowerCase());
+        if (activeTasks.length === 0)
+            return randomPick;
+        // Find configured topics that match keywords in active agent tasks
+        const taskText = activeTasks.join(" ");
+        const matched = topics.filter(t => taskText.includes(t.toLowerCase()));
+        if (matched.length > 0) {
+            const chosen = matched[Math.floor(Math.random() * matched.length)];
+            debugLog(`[WebScholar] 🐝 Task-aware topic: "${chosen}" (matched from active agent tasks)`);
+            return chosen;
+        }
+    }
+    catch (err) {
+        debugLog(`[WebScholar] Task-aware selection failed (non-fatal), using random: ${err}`);
+    }
+    return randomPick;
+}
+// ─── Core Pipeline ───────────────────────────────────────────
+/**
+ * Runs the Web Scholar pipeline:
+ * 1. Picks a topic (task-aware when Hivemind is on)
+ * 2. Registers on the Hivemind Radar
+ * 3. Searches Brave for recent articles
+ * 4. Scrapes articles as markdown using Firecrawl
+ * 5. Summarizes the findings via LLM
+ * 6. Injects the summary directly into Prism's semantic ledger
+ * 7. Broadcasts a Telepathy alert to active agents
+ */
+let isRunning = false;
+export async function runWebScholar() {
+    if (isRunning) {
+        debugLog("[WebScholar] Skipped: already running");
+        return;
+    }
+    isRunning = true;
+    const tracer = getTracer();
+    const span = tracer.startSpan("background.web_scholar");
+    try {
+        const useFreeFallback = !BRAVE_API_KEY || !FIRECRAWL_API_KEY;
+        if (!PRISM_SCHOLAR_TOPICS || PRISM_SCHOLAR_TOPICS.length === 0) {
+            debugLog("[WebScholar] Skipped: No topics configured in PRISM_SCHOLAR_TOPICS");
+            span.setAttribute("scholar.skipped_reason", "no_topics");
+            return;
+        }
+        // 1. Pick a topic (task-aware when Hivemind is active)
+        const topic = await selectTopic();
+        if (!topic) {
+            span.setAttribute("scholar.skipped_reason", "no_topics");
+            return;
+        }
+        debugLog(`[WebScholar] 🧠 Starting research on topic: "${topic}"`);
+        span.setAttribute("scholar.topic", topic);
+        // 2. Register on Hivemind Radar
+        await hivemindRegister(topic);
+        // 3. Search for articles
+        await hivemindHeartbeat(`Searching for: ${topic}`);
+        let urls = [];
+        if (useFreeFallback) {
+            debugLog("[WebScholar] API keys missing, falling back to Local Free Search (Yahoo + Readability)");
+            const ddgResults = await searchYahooFree(topic, PRISM_SCHOLAR_MAX_ARTICLES_PER_RUN);
+            urls = ddgResults.map(r => r.url).filter(Boolean);
+        }
+        else {
+            const braveResponse = await performWebSearchRaw(topic, PRISM_SCHOLAR_MAX_ARTICLES_PER_RUN);
+            const braveData = JSON.parse(braveResponse);
+            urls = (braveData.web?.results || []).map((r) => r.url).filter(Boolean);
+        }
+        if (urls.length === 0) {
+            debugLog(`[WebScholar] No articles found for "${topic}"`);
+            span.setAttribute("scholar.skipped_reason", "no_search_results");
+            return;
+        }
+        debugLog(`[WebScholar] Found ${urls.length} articles. Scraping...`);
+        span.setAttribute("scholar.articles_found", urls.length);
+        // 4. Scrape each URL
+        await hivemindHeartbeat(`Scraping ${urls.length} articles on: ${topic}`);
+        const scrapedTexts = [];
+        for (const url of urls) {
+            if (useFreeFallback) {
+                try {
+                    debugLog(`[WebScholar] Scraping local fallback: ${url}`);
+                    const article = await scrapeArticleLocal(url);
+                    const trimmed = article.content.slice(0, 15_000);
+                    scrapedTexts.push(`Source: ${url}\nTitle: ${article.title}\n\n${trimmed}\n\n---\n`);
+                }
+                catch (err) {
+                    console.error(`[WebScholar] Failed to locally scrape ${url}:`, err);
+                }
+            }
+            else {
+                try {
+                    debugLog(`[WebScholar] Scraping Firecrawl: ${url}`);
+                    const scrapeRes = await fetch("https://api.firecrawl.dev/v1/scrape", {
+                        method: "POST",
+                        headers: {
+                            "Content-Type": "application/json",
+                            "Authorization": `Bearer ${FIRECRAWL_API_KEY}`
+                        },
+                        body: JSON.stringify({
+                            url,
+                            formats: ["markdown"],
+                        })
+                    });
+                    if (!scrapeRes.ok) {
+                        console.error(`[WebScholar] Firecrawl failed for ${url}: ${scrapeRes.status}`);
+                        continue;
+                    }
+                    const result = (await scrapeRes.json());
+                    if (result.success && result.data?.markdown) {
+                        const trimmed = result.data.markdown.slice(0, 15_000);
+                        scrapedTexts.push(`Source: ${url}\n\n${trimmed}\n\n---\n`);
+                    }
+                }
+                catch (err) {
+                    console.error(`[WebScholar] Failed to scrape ${url}:`, err);
+                }
+            }
+        }
+        if (scrapedTexts.length === 0) {
+            debugLog(`[WebScholar] Could not extract markdown from any articles.`);
+            span.setAttribute("scholar.skipped_reason", "all_scrapes_failed");
+            return;
+        }
+        span.setAttribute("scholar.articles_scraped", scrapedTexts.length);
+        // 5. Summarize findings using LLM
+        await hivemindHeartbeat(`Synthesizing ${scrapedTexts.length} articles on: ${topic}`);
+        debugLog(`[WebScholar] Summarizing ${scrapedTexts.length} articles...`);
+        const combinedText = scrapedTexts.join("\n");
+        const prompt = `You are an AI research assistant. You have been asked to research the topic: "${topic}".
+Read the following scraped web articles and write a comprehensive, markdown-formatted report summarizing the key findings, trends, and actionable insights. Focus heavily on facts, data, and actual content. Do NOT just list the articles. Synthesize the information.
+
+### Scraped Articles:
+${combinedText}`;
+        const llm = getLLMProvider();
+        const summary = await llm.generateText(prompt);
+        // 6. Inject the summary back into Prism memory
+        await hivemindHeartbeat(`Saving research to ledger: ${topic}`);
+        const storage = await getStorage();
+        await storage.saveLedger({
+            id: randomUUID(),
+            project: "prism-scholar",
+            conversation_id: "scholar-bg-" + Date.now(),
+            user_id: PRISM_USER_ID,
+            role: "scholar",
+            summary: `Autonomous Web Scholar Research: ${topic}\n\n${summary}`,
+            keywords: [topic, "research", "autonomous", "scholar"],
+            event_type: "learning",
+            importance: 7,
+            created_at: new Date().toISOString()
+        });
+        debugLog(`[WebScholar] ✅ Research complete and saved to ledger under project 'prism-scholar'.`);
+        span.setAttribute("scholar.success", true);
+        // 7. Broadcast Telepathy alert to active agents
+        await hivemindBroadcast(topic, scrapedTexts.length);
+    }
+    catch (err) {
+        console.error("[WebScholar] Pipeline failed:", err);
+        span.setAttribute("scholar.error", String(err));
+    }
+    finally {
+        await hivemindIdle();
+        isRunning = false;
+        span.end();
+    }
+}

package/dist/sdm/sdmDecoder.js

@@ -0,0 +1,75 @@
+import { getStorage } from "../storage/index.js";
+import { getDefaultCompressor, deserialize } from "../utils/turboquant.js";
+import { debugLog } from "../utils/logger.js";
+/**
+ * Perform a fast JS-space Hamming distance scan across TurboQuant compressed embeddings.
+ * Used exclusively for decoding SDM superposed target vectors back into ledger entries.
+ */
+export async function decodeSdmVector(project, targetVector, matchCount = 5, similarityThreshold = 0.55) {
+    const t0 = performance.now();
+    const storage = await getStorage();
+    // 1. Fetch all compressed embeddings for the active project
+    const embeddings = await storage.getAllProjectEmbeddings(project);
+    if (embeddings.length === 0) {
+        debugLog(`[SdmDecoder] No embeddings found for project ${project}`);
+        return [];
+    }
+    // 2. Compress the target Float32Array into a 96-byte Uint8Array using TurboQuant
+    const compressor = getDefaultCompressor();
+    // compressor.compress takes number[]
+    const queryArray = Array.from(targetVector); // V8 optimized array conversion
+    const compressedTarget = compressor.compress(queryArray);
+    const targetQjl = compressedTarget.qjlSigns;
+    // Ensure byte length aligns with 32-bit words for fast XOR
+    // We copy the target Uint8Array into a perfectly aligned Int32Array buffer
+    const wordCount = targetQjl.length / 4;
+    const targetWords = new Int32Array(wordCount);
+    new Uint8Array(targetWords.buffer).set(targetQjl);
+    // 3. Scan all DB embeddings and compute Hamming distance on QJL signs
+    const matches = [];
+    // Pre-allocate scratchpad Int32Array to avoid GC pressure during decoding sweeps
+    const dbWords = new Int32Array(wordCount);
+    const dbBytes = new Uint8Array(dbWords.buffer);
+    for (const entry of embeddings) {
+        try {
+            const dbBuf = Buffer.from(entry.embedding_compressed.replace(/^v1\./, ""), "base64");
+            const compressedDb = deserialize(dbBuf);
+            const dbQjl = compressedDb.qjlSigns;
+            if (dbQjl.length !== targetQjl.length)
+                continue;
+            // Zero-allocation copy into the scratchpad
+            dbBytes.set(dbQjl);
+            let hammingDistance = 0;
+            for (let i = 0; i < wordCount; i++) {
+                // Bitwise XOR to find mismatched bits
+                let xor = targetWords[i] ^ dbWords[i];
+                // Brian Kernighan's algorithm or V8 Math.clz32 trick isn't natively popcnt,
+                // but simple bitwise popcount is fast enough in V8:
+                xor = xor - ((xor >>> 1) & 0x55555555);
+                xor = (xor & 0x33333333) + ((xor >>> 2) & 0x33333333);
+                hammingDistance += (((xor + (xor >>> 4)) & 0x0F0F0F0F) * 0x01010101) >>> 24;
+            }
+            // Convert distance to standard similarity score (1.0 = exact match)
+            const totalBits = targetQjl.byteLength * 8;
+            const similarity = 1 - (hammingDistance / totalBits);
+            if (similarity >= similarityThreshold) {
+                matches.push({
+                    id: entry.id,
+                    summary: entry.summary,
+                    distance: hammingDistance,
+                    similarity
+                });
+            }
+        }
+        catch (err) {
+            // Ignore malformed embeddings
+            continue;
+        }
+    }
+    // 4. Sort by highest similarity
+    matches.sort((a, b) => b.similarity - a.similarity);
+    const topMatches = matches.slice(0, matchCount);
+    const t1 = performance.now();
+    debugLog(`[SdmDecoder] Decoded target vector against ${embeddings.length} entries in ${(t1 - t0).toFixed(2)}ms`);
+    return topMatches;
+}

package/dist/sdm/sdmEngine.js

@@ -0,0 +1,158 @@
+import { PRISM_DEFAULT_CONFIG, getDefaultCompressor } from '../utils/turboquant.js';
+// M = 10,000 hard locations per project
+const SDM_M = 10000;
+// D_addr = 768 bits (binary QJL string length), represented as 24 Uint32s
+const D_ADDR_UINT32 = PRISM_DEFAULT_CONFIG.d / 32;
+// The L1 radius threshold for "activation" (how many bits can differ)
+// For 768 bits, picking a threshold like 300 gives us sparse activation
+const ACTIVATION_RADIUS = 300;
+// We use PRNG seeded deterministically to generate the initial hard location addresses
+// This ensures that restarts produce the exact same Kanerva address space
+class PRNG {
+    seed;
+    constructor(seed) {
+        this.seed = seed;
+    }
+    nextUInt32() {
+        this.seed = Math.imul(this.seed ^ (this.seed >>> 15), 1 | this.seed);
+        this.seed ^= this.seed + Math.imul(this.seed ^ (this.seed >>> 7), 61 | this.seed);
+        const v = ((this.seed ^ (this.seed >>> 14)) >>> 0);
+        return v;
+    }
+}
+/**
+ * Fast Hamming Distance over Uint32 arrays
+ */
+export function hammingDistance(a, b) {
+    let sum = 0;
+    for (let i = 0; i < a.length; i++) {
+        let xor = a[i] ^ b[i];
+        // 32-bit popcount trick
+        xor -= ((xor >>> 1) & 0x55555555);
+        xor = (xor & 0x33333333) + ((xor >>> 2) & 0x33333333);
+        xor = (xor + (xor >>> 4)) & 0x0F0F0F0F;
+        sum += Math.imul(xor, 0x01010101) >>> 24;
+    }
+    return sum;
+}
+export class SparseDistributedMemory {
+    // Hard Locations: Addresses (M x 24 uint32)
+    addresses;
+    // Hard Locations: Counters (M x 768 float32)
+    counters;
+    constructor(seed = 42) {
+        this.addresses = new Array(SDM_M);
+        this.counters = new Array(SDM_M);
+        const prng = new PRNG(seed);
+        for (let i = 0; i < SDM_M; i++) {
+            const addr = new Uint32Array(D_ADDR_UINT32);
+            for (let j = 0; j < D_ADDR_UINT32; j++) {
+                addr[j] = prng.nextUInt32();
+            }
+            this.addresses[i] = addr;
+            this.counters[i] = new Float32Array(PRISM_DEFAULT_CONFIG.d);
+        }
+    }
+    /** Convert TurboQuant QJL bytes into Uint32Array for fast bit math */
+    blobToAddress(blob) {
+        const qjl = blob.qjlSigns; // Uint8Array of length 96 (768 bits)
+        const view = new DataView(qjl.buffer, qjl.byteOffset, qjl.byteLength);
+        const addr = new Uint32Array(D_ADDR_UINT32);
+        for (let i = 0; i < D_ADDR_UINT32; i++) {
+            // Needs little endian to match bit alignment expectations safely
+            addr[i] = view.getUint32(i * 4, true);
+        }
+        return addr;
+    }
+    /**
+     * Write a dense vector into the memory by routing it to activated counters
+     */
+    write(vector, k = 20) {
+        const compressor = getDefaultCompressor();
+        const blob = compressor.compress(Array.from(vector));
+        const address = this.blobToAddress(blob);
+        const activated = this.getTopK(address, k);
+        for (const idx of activated) {
+            const c = this.counters[idx];
+            for (let j = 0; j < PRISM_DEFAULT_CONFIG.d; j++) {
+                c[j] += vector[j];
+            }
+        }
+    }
+    read(queryVector, k = 20) {
+        const compressor = getDefaultCompressor();
+        const blob = compressor.compress(Array.from(queryVector));
+        const address = this.blobToAddress(blob);
+        const result = new Float32Array(PRISM_DEFAULT_CONFIG.d);
+        const activated = this.getTopK(address, k);
+        for (const idx of activated) {
+            const c = this.counters[idx];
+            for (let j = 0; j < PRISM_DEFAULT_CONFIG.d; j++) {
+                result[j] += c[j];
+            }
+        }
+        return this.l2Normalize(result);
+    }
+    getTopK(address, k) {
+        // Array of (dist, index)
+        const dists = new Array(SDM_M);
+        for (let i = 0; i < SDM_M; i++) {
+            dists[i] = { d: hammingDistance(address, this.addresses[i]), i };
+        }
+        // Partial sort to find top K
+        dists.sort((a, b) => a.d - b.d);
+        const result = new Array(k);
+        for (let i = 0; i < k; i++) {
+            result[i] = dists[i].i;
+        }
+        return result;
+    }
+    l2Normalize(vec) {
+        let sum = 0;
+        for (let i = 0; i < vec.length; i++) {
+            sum += vec[i] * vec[i];
+        }
+        if (sum === 0)
+            return vec;
+        const mag = Math.sqrt(sum);
+        for (let i = 0; i < vec.length; i++) {
+            vec[i] /= mag;
+        }
+        return vec;
+    }
+    /**
+     * Export the entire 10k x 768 counter matrix as a single 1D Float32Array
+     * for binary serialization to SQLite BLOB.
+     */
+    exportState() {
+        const state = new Float32Array(SDM_M * PRISM_DEFAULT_CONFIG.d);
+        for (let i = 0; i < SDM_M; i++) {
+            state.set(this.counters[i], i * PRISM_DEFAULT_CONFIG.d);
+        }
+        return state;
+    }
+    /**
+     * Import a previously serialized 1D Float32Array matrix back into
+     * the 2D counters array.
+     */
+    importState(state) {
+        if (state.length !== SDM_M * PRISM_DEFAULT_CONFIG.d) {
+            throw new Error(`Invalid SDM state size: expected ${SDM_M * PRISM_DEFAULT_CONFIG.d}, got ${state.length}`);
+        }
+        for (let i = 0; i < SDM_M; i++) {
+            // Subarray creates a fast view over the underlying buffer
+            this.counters[i] = state.subarray(i * PRISM_DEFAULT_CONFIG.d, (i + 1) * PRISM_DEFAULT_CONFIG.d);
+        }
+    }
+}
+// Global Singleton per Project in memory
+const _sdmInstances = new Map();
+export function getSdmEngine(projectId) {
+    if (!_sdmInstances.has(projectId)) {
+        _sdmInstances.set(projectId, new SparseDistributedMemory());
+    }
+    return _sdmInstances.get(projectId);
+}
+export function getAllActiveSdmProjects() {
+    return Array.from(_sdmInstances.keys());
+}