specmem-hardwicksoftware 3.5.99

package/embedding-sandbox/server-batch.mjs

@@ -0,0 +1,228 @@
+/**
+ * SANDBOXED EMBEDDING SERVER - BATCH SUPPORT
+ *
+ * Adds batch embedding to enable 100+/s throughput
+ */
+
+import { pipeline, env } from '@huggingface/transformers';
+import { createServer } from 'net';
+import { existsSync, unlinkSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+// Machine-unique shared socket path
+const getMachineSocketPath = () => {
+  const userId = process.getuid ? process.getuid() : 'default';
+  return join(tmpdir(), `specmem-embed-${userId}.sock`);
+};
+
+const SOCKET_PATH = process.env.SOCKET_PATH || getMachineSocketPath();
+const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2';
+
+let NATIVE_DIM = null;
+let TARGET_DIM = null;
+
+const CACHE_DIR = process.env.HF_HOME || '/home/embed/.cache/huggingface';
+env.cacheDir = CACHE_DIR;
+
+console.log('[Sandbox] Starting BATCH embedding server...');
+console.log('[Sandbox] Socket:', SOCKET_PATH);
+
+let extractor = null;
+let modelReady = false;
+
+async function loadModel() {
+  try {
+    console.log('[Sandbox] Loading model...');
+    extractor = await pipeline('feature-extraction', MODEL_NAME, {
+      local_files_only: true,
+      device: 'cpu'
+    });
+    modelReady = true;
+
+    const testOutput = await extractor('test', { pooling: 'mean', normalize: true });
+    NATIVE_DIM = testOutput.data.length;
+    TARGET_DIM = NATIVE_DIM;
+
+    console.log('[Sandbox] Model ready. Dimension:', NATIVE_DIM);
+    console.log('[Sandbox] BATCH support enabled for high throughput');
+  } catch (error) {
+    console.error('[Sandbox] FATAL:', error.message);
+  }
+}
+
+function scaleEmbedding(embedding, targetDim) {
+  const srcDim = embedding.length;
+  if (srcDim === targetDim) return embedding;
+
+  const result = new Float64Array(targetDim);
+  if (targetDim < srcDim) {
+    const ratio = srcDim / targetDim;
+    for (let i = 0; i < targetDim; i++) {
+      const start = Math.floor(i * ratio);
+      const end = Math.floor((i + 1) * ratio);
+      let sum = 0;
+      for (let j = start; j < end; j++) sum += embedding[j];
+      result[i] = sum / (end - start);
+    }
+  } else {
+    const ratio = (srcDim - 1) / (targetDim - 1);
+    for (let i = 0; i < targetDim; i++) {
+      const srcIdx = i * ratio;
+      const low = Math.floor(srcIdx);
+      const high = Math.min(low + 1, srcDim - 1);
+      const frac = srcIdx - low;
+      result[i] = embedding[low] * (1 - frac) + embedding[high] * frac;
+    }
+  }
+
+  let norm = 0;
+  for (let i = 0; i < targetDim; i++) norm += result[i] * result[i];
+  norm = Math.sqrt(norm);
+  if (norm > 0) for (let i = 0; i < targetDim; i++) result[i] /= norm;
+  return Array.from(result);
+}
+
+async function generateEmbedding(text) {
+  if (!modelReady) throw new Error('Model not loaded');
+  const truncated = text.slice(0, 1000);
+  const output = await extractor(truncated, { pooling: 'mean', normalize: true });
+  let embedding = Array.from(output.data);
+  if (TARGET_DIM && embedding.length !== TARGET_DIM) {
+    embedding = scaleEmbedding(embedding, TARGET_DIM);
+  }
+  return embedding;
+}
+
+// BATCH embedding - process multiple texts at once
+async function generateBatchEmbeddings(texts) {
+  if (!modelReady) throw new Error('Model not loaded');
+
+  const results = [];
+  // Process in parallel chunks for efficiency
+  const CHUNK_SIZE = 10;
+
+  for (let i = 0; i < texts.length; i += CHUNK_SIZE) {
+    const chunk = texts.slice(i, i + CHUNK_SIZE);
+    const promises = chunk.map(async (text) => {
+      try {
+        const truncated = text.slice(0, 1000);
+        const output = await extractor(truncated, { pooling: 'mean', normalize: true });
+        let embedding = Array.from(output.data);
+        if (TARGET_DIM && embedding.length !== TARGET_DIM) {
+          embedding = scaleEmbedding(embedding, TARGET_DIM);
+        }
+        return { embedding, error: null };
+      } catch (e) {
+        return { embedding: null, error: e.message };
+      }
+    });
+
+    const chunkResults = await Promise.all(promises);
+    results.push(...chunkResults);
+  }
+
+  return results;
+}
+
+function handleRequest(socket) {
+  let buffer = '';
+
+  socket.on('data', async (data) => {
+    buffer += data.toString();
+    const newlineIndex = buffer.indexOf('\n');
+    if (newlineIndex === -1) return;
+
+    const requestJson = buffer.slice(0, newlineIndex);
+    buffer = buffer.slice(newlineIndex + 1);
+
+    try {
+      const request = JSON.parse(requestJson);
+
+      // Health check
+      if (request.type === 'health') {
+        socket.write(JSON.stringify({
+          status: modelReady ? 'healthy' : 'loading',
+          model: MODEL_NAME,
+          native_dimensions: NATIVE_DIM,
+          target_dimensions: TARGET_DIM,
+          batch_support: true
+        }) + '\n');
+        return;
+      }
+
+      // Set dimension
+      if (request.type === 'set_dimension') {
+        if (request.dimension && typeof request.dimension === 'number' && request.dimension > 0) {
+          TARGET_DIM = request.dimension;
+          socket.write(JSON.stringify({ status: 'ok', dimension: TARGET_DIM }) + '\n');
+        } else {
+          socket.write(JSON.stringify({ error: 'Invalid dimension' }) + '\n');
+        }
+        return;
+      }
+
+      // BATCH embedding - new!
+      if (request.type === 'batch_embed') {
+        if (!Array.isArray(request.texts)) {
+          socket.write(JSON.stringify({ error: 'texts must be array' }) + '\n');
+          return;
+        }
+
+        const startTime = Date.now();
+        const results = await generateBatchEmbeddings(request.texts);
+        const elapsed = Date.now() - startTime;
+
+        socket.write(JSON.stringify({
+          embeddings: results.map(r => r.embedding),
+          errors: results.map(r => r.error),
+          count: request.texts.length,
+          elapsed_ms: elapsed,
+          rate: (request.texts.length / (elapsed / 1000)).toFixed(1)
+        }) + '\n');
+        return;
+      }
+
+      // Single embed (backwards compatible)
+      if (request.type === 'embed') {
+        if (!request.text || typeof request.text !== 'string') {
+          socket.write(JSON.stringify({ error: 'Missing text' }) + '\n');
+          return;
+        }
+        const embedding = await generateEmbedding(request.text);
+        socket.write(JSON.stringify({
+          embedding,
+          dimensions: embedding.length
+        }) + '\n');
+        return;
+      }
+
+      socket.write(JSON.stringify({ error: 'Unknown request type' }) + '\n');
+
+    } catch (error) {
+      socket.write(JSON.stringify({ error: error.message }) + '\n');
+    }
+  });
+
+  socket.on('error', () => {});
+}
+
+async function main() {
+  await loadModel();
+
+  if (existsSync(SOCKET_PATH)) unlinkSync(SOCKET_PATH);
+
+  const server = createServer(handleRequest);
+  server.listen(SOCKET_PATH, () => {
+    console.log('[Sandbox] Listening on:', SOCKET_PATH);
+  });
+
+  process.on('SIGTERM', () => {
+    console.log('[Sandbox] Shutting down...');
+    server.close();
+    if (existsSync(SOCKET_PATH)) unlinkSync(SOCKET_PATH);
+    process.exit(0);
+  });
+}
+
+main();

package/embedding-sandbox/server.mjs

@@ -0,0 +1,389 @@
+/**
+ * SANDBOXED EMBEDDING SERVER - DYNAMIC DIMENSIONS
+ *
+ * This server runs inside a Docker container with NO NETWORK ACCESS.
+ * It can ONLY:
+ *   - Receive text via Unix socket
+ *   - Return embedding vectors at DATABASE dimension
+ *   - Read the pre-downloaded model weights
+ *
+ * DYNAMIC DIMENSIONS:
+ *   - Native dimension detected from model at load time
+ *   - Target dimension queried from database or set via protocol
+ *   - Embeddings scaled to match database dimension
+ *
+ * Security measures:
+ *   - Docker --network none
+ *   - Read-only root filesystem
+ *   - No capabilities (--cap-drop ALL)
+ *   - Runs as non-root user
+ *   - Unix socket only (no TCP)
+ */
+
+import { pipeline, env } from '@huggingface/transformers';
+import { createServer } from 'net';
+import { existsSync, unlinkSync } from 'fs';
+
+// Machine-unique shared socket path - embeddings are stateless so sharing is safe
+// Pattern: /tmp/specmem-embed-{uid}.sock
+// Single embedding server per user, shared across all projects for efficiency
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+const getMachineSocketPath = () => {
+  // Priority order for socket path:
+  // 1. SPECMEM_EMBEDDING_SOCKET - explicit embedding socket path
+  if (process.env.SPECMEM_EMBEDDING_SOCKET) {
+    return process.env.SPECMEM_EMBEDDING_SOCKET;
+  }
+
+  // 2. SPECMEM_SOCKET_DIR - socket directory with embeddings.sock filename
+  if (process.env.SPECMEM_SOCKET_DIR) {
+    return join(process.env.SPECMEM_SOCKET_DIR, 'embeddings.sock');
+  }
+
+  // 3. SPECMEM_PROJECT_PATH - project-specific socket path
+  if (process.env.SPECMEM_PROJECT_PATH) {
+    return join(process.env.SPECMEM_PROJECT_PATH, 'specmem', 'sockets', 'embeddings.sock');
+  }
+
+  // 4. Fallback: machine-unique shared socket (legacy behavior)
+  // Use UID for machine-unique but shared socket (not per-project)
+  // Embeddings are stateless - one server can serve all projects
+  const userId = process.getuid ? process.getuid() : 'default';
+  return join(tmpdir(), `specmem-embed-${userId}.sock`);
+};
+
+const SOCKET_PATH = process.env.SOCKET_PATH || getMachineSocketPath();
+const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2';
+
+// Dynamic dimensions - detected from model and database
+let NATIVE_DIM = null;
+let TARGET_DIM = null;
+
+// Set cache directory explicitly - must match download-model.mjs
+const CACHE_DIR = process.env.HF_HOME || '/home/embed/.cache/huggingface';
+env.cacheDir = CACHE_DIR;
+
+console.log('[Sandbox] Starting air-gapped embedding server...');
+console.log('[Sandbox] NO NETWORK ACCESS - this is intentional');
+console.log('[Sandbox] Socket path:', SOCKET_PATH);
+
+// Load the model (from local cache only - no network)
+let extractor = null;
+let modelReady = false;
+let loadError = null;
+
+async function loadModel() {
+  try {
+    console.log('[Sandbox] Loading model from local cache...');
+
+    // Force local-only mode - will fail if model not pre-downloaded
+    extractor = await pipeline('feature-extraction', MODEL_NAME, {
+      // Use local cache only - no downloads allowed
+      local_files_only: true,
+      // Use CPU only (safer, no GPU driver access)
+      device: 'cpu'
+    });
+
+    modelReady = true;
+
+    // Detect native dimension by running a test embedding
+    const testOutput = await extractor('test', { pooling: 'mean', normalize: true });
+    NATIVE_DIM = testOutput.data.length;
+    TARGET_DIM = NATIVE_DIM; // Default to native, can be set via protocol
+
+    console.log('[Sandbox] Model loaded successfully. Native dimension:', NATIVE_DIM);
+    console.log('[Sandbox] Target dimension:', TARGET_DIM, '(can be set via protocol)');
+    console.log('[Sandbox] Ready to accept embedding requests.');
+  } catch (error) {
+    loadError = error.message;
+    console.error('[Sandbox] FATAL: Failed to load model:', error.message);
+    console.error('[Sandbox] Did you run "npm run download-model" first?');
+    // Don't exit - let the server return errors instead
+  }
+}
+
+// Scale embedding to target dimension (up or down)
+function scaleEmbedding(embedding, targetDim) {
+  const srcDim = embedding.length;
+  if (srcDim === targetDim) return embedding;
+
+  const result = new Float64Array(targetDim);
+
+  if (targetDim < srcDim) {
+    // Downscale: average neighboring values
+    const ratio = srcDim / targetDim;
+    for (let i = 0; i < targetDim; i++) {
+      const start = Math.floor(i * ratio);
+      const end = Math.floor((i + 1) * ratio);
+      let sum = 0;
+      for (let j = start; j < end; j++) {
+        sum += embedding[j];
+      }
+      result[i] = sum / (end - start);
+    }
+  } else {
+    // Upscale: linear interpolation
+    const ratio = (srcDim - 1) / (targetDim - 1);
+    for (let i = 0; i < targetDim; i++) {
+      const srcIdx = i * ratio;
+      const low = Math.floor(srcIdx);
+      const high = Math.min(low + 1, srcDim - 1);
+      const frac = srcIdx - low;
+      result[i] = embedding[low] * (1 - frac) + embedding[high] * frac;
+    }
+  }
+
+  // Normalize to unit length
+  let norm = 0;
+  for (let i = 0; i < targetDim; i++) {
+    norm += result[i] * result[i];
+  }
+  norm = Math.sqrt(norm);
+  if (norm > 0) {
+    for (let i = 0; i < targetDim; i++) {
+      result[i] /= norm;
+    }
+  }
+
+  return Array.from(result);
+}
+
+// Generate embedding for text, scaled to target dimension
+async function generateEmbedding(text) {
+  if (!modelReady) {
+    throw new Error('Model not loaded: ' + (loadError || 'unknown error'));
+  }
+
+  // Truncate to model's max length (256 tokens ~= 1000 chars)
+  const truncated = text.slice(0, 1000);
+
+  // Generate embedding at native dimension
+  const output = await extractor(truncated, {
+    pooling: 'mean',
+    normalize: true
+  });
+
+  // Convert to array
+  let embedding = Array.from(output.data);
+
+  // Scale to target dimension if different
+  if (TARGET_DIM && embedding.length !== TARGET_DIM) {
+    embedding = scaleEmbedding(embedding, TARGET_DIM);
+  }
+
+  return embedding;
+}
+
+// Handle incoming requests
+function handleRequest(socket) {
+  let buffer = '';
+
+  socket.on('data', async (data) => {
+    buffer += data.toString();
+
+    // Check for complete JSON (newline-delimited)
+    const newlineIndex = buffer.indexOf('\n');
+    if (newlineIndex === -1) return;
+
+    const requestJson = buffer.slice(0, newlineIndex);
+    buffer = buffer.slice(newlineIndex + 1);
+
+    try {
+      const request = JSON.parse(requestJson);
+
+      // BACKWARDS COMPATIBILITY: Support Python-style requests
+      // {"stats": true} -> treat like {"type": "health"}
+      if (request.stats === true) {
+        socket.write(JSON.stringify({
+          status: modelReady ? 'healthy' : 'loading',
+          stats: { model: MODEL_NAME, ready: modelReady },
+          model: MODEL_NAME,
+          native_dimensions: NATIVE_DIM,
+          target_dimensions: TARGET_DIM,
+          dynamic: true
+        }) + '\n');
+        return;
+      }
+
+      // {"text": "..."} without type -> treat like {"type": "embed", "text": "..."}
+      if (!request.type && request.text && typeof request.text === 'string') {
+        request.type = 'embed';
+      }
+
+      // {"texts": [...]} batch format -> handle ALL texts, return {"embeddings": [[...], ...]}
+      if (!request.type && request.texts && Array.isArray(request.texts)) {
+        console.log('[Sandbox] Batch request - processing', request.texts.length, 'texts');
+
+        // Send processing heartbeat
+        socket.write(JSON.stringify({
+          status: 'processing',
+          count: request.texts.length
+        }) + '\n');
+
+        try {
+          const embeddings = [];
+          for (const text of request.texts) {
+            if (typeof text === 'string' && text.trim()) {
+              const embedding = await generateEmbedding(text);
+              embeddings.push(embedding);
+            } else {
+              // Invalid text - use zeros
+              embeddings.push(new Array(TARGET_DIM || NATIVE_DIM).fill(0));
+            }
+          }
+
+          socket.write(JSON.stringify({
+            embeddings,
+            dimensions: embeddings[0]?.length || TARGET_DIM || NATIVE_DIM,
+            count: embeddings.length,
+            native_dimensions: NATIVE_DIM,
+            target_dimensions: TARGET_DIM,
+            model: MODEL_NAME
+          }) + '\n');
+        } catch (error) {
+          socket.write(JSON.stringify({ error: error.message, batch: true }) + '\n');
+        }
+        return;
+      }
+
+      // {"query": "..."} format from MiniCOT/other clients -> treat as embed
+      if (!request.type && request.query && typeof request.query === 'string') {
+        request.type = 'embed';
+        request.text = request.query;
+        console.log('[Sandbox] Query format - converting to embed');
+      }
+
+      // Handle set_dimension request
+      if (request.type === 'set_dimension') {
+        if (request.dimension && typeof request.dimension === 'number' && request.dimension > 0) {
+          TARGET_DIM = request.dimension;
+          console.log('[Sandbox] Target dimension set to:', TARGET_DIM);
+          socket.write(JSON.stringify({ status: 'ok', dimension: TARGET_DIM }) + '\n');
+        } else {
+          socket.write(JSON.stringify({ error: 'Invalid dimension value' }) + '\n');
+        }
+        return;
+      }
+
+      // Handle get_dimension request
+      if (request.type === 'get_dimension') {
+        socket.write(JSON.stringify({
+          native_dimensions: NATIVE_DIM,
+          target_dimensions: TARGET_DIM
+        }) + '\n');
+        return;
+      }
+
+      // Handle health check
+      if (request.type === 'health') {
+        socket.write(JSON.stringify({
+          status: modelReady ? 'healthy' : 'loading',
+          model: MODEL_NAME,
+          native_dimensions: NATIVE_DIM,
+          target_dimensions: TARGET_DIM,
+          dynamic: true
+        }) + '\n');
+        return;
+      }
+
+      if (request.type !== 'embed') {
+        // More descriptive error with request info for debugging
+        const reqKeys = Object.keys(request).join(', ');
+        console.error('[Sandbox] Unknown request type:', request.type, '- keys:', reqKeys);
+        socket.write(JSON.stringify({
+          error: `Unknown request type: ${request.type || 'undefined'}`,
+          received_keys: reqKeys,
+          hint: 'Expected: {type:"embed",text:"..."} or {text:"..."} or {stats:true} or {type:"health"}',
+          requestId: request.requestId
+        }) + '\n');
+        return;
+      }
+
+      if (!request.text || typeof request.text !== 'string') {
+        const reqKeys = Object.keys(request).join(', ');
+        console.error('[Sandbox] Missing text field - keys:', reqKeys);
+        socket.write(JSON.stringify({
+          error: 'Missing or invalid text field',
+          received_keys: reqKeys,
+          hint: 'Include "text" field with string value',
+          requestId: request.requestId
+        }) + '\n');
+        return;
+      }
+
+      // Send "processing" heartbeat immediately so client knows we're working
+      // This enables idle-based timeouts - client resets timer on any data
+      // Include requestId for persistent socket matching
+      socket.write(JSON.stringify({
+        status: 'processing',
+        text_length: request.text.length,
+        requestId: request.requestId
+      }) + '\n');
+
+      // Generate embedding (scaled to target dimension)
+      const embedding = await generateEmbedding(request.text);
+
+      socket.write(JSON.stringify({
+        embedding,
+        dimensions: embedding.length,
+        native_dimensions: NATIVE_DIM,
+        target_dimensions: TARGET_DIM,
+        model: MODEL_NAME,
+        dynamic: true,
+        requestId: request.requestId
+      }) + '\n');
+
+    } catch (error) {
+      socket.write(JSON.stringify({ error: error.message }) + '\n');
+    }
+  });
+
+  socket.on('error', (err) => {
+    // Silently ignore EPIPE/connection reset - these are harmless
+    // Common causes: health checks, clients that disconnect early
+    if (err.code === 'EPIPE' || err.code === 'ECONNRESET' || err.message.includes('EPIPE')) {
+      return; // Silent ignore
+    }
+    console.error('[Sandbox] Socket error:', err.message);
+  });
+}
+
+// Start the server
+async function main() {
+  // Load model first
+  await loadModel();
+
+  // Clean up existing socket
+  if (existsSync(SOCKET_PATH)) {
+    unlinkSync(SOCKET_PATH);
+  }
+
+  // Create Unix socket server
+  const server = createServer(handleRequest);
+
+  server.listen(SOCKET_PATH, () => {
+    console.log('[Sandbox] Listening on Unix socket:', SOCKET_PATH);
+  });
+
+  server.on('error', (err) => {
+    console.error('[Sandbox] Server error:', err);
+    process.exit(1);
+  });
+
+  // Graceful shutdown
+  process.on('SIGTERM', () => {
+    console.log('[Sandbox] Shutting down...');
+    server.close();
+    if (existsSync(SOCKET_PATH)) {
+      unlinkSync(SOCKET_PATH);
+    }
+    process.exit(0);
+  });
+}
+
+main().catch(err => {
+  console.error('[Sandbox] Fatal error:', err);
+  process.exit(1);
+});

package/embedding-sandbox/specmem/sockets/claude-input-state.json

@@ -0,0 +1 @@
+{"lastInput":" NO AI ATTRIBUTION PERMITTED! Bet that shit up lil nig, deploy.sh patch -> Wipe github commit history, update all github SOURCE files to curent -- do not replace license changelog svgs or pages, your goal is to make the girhub have updated files but appear as if first commit, ensure no ai attribution, wipe commit and push history, wipe release history and make the 3.5.3 the new release","lastInputTime":1769974346692,"buffer":"NO AI ATTRIBUTION PERMITTED! Bet that shit up lil nig, deploy.sh patch -> Wipe github commit history, update all github SOURCE files to curent -- do not replace license changelog svgs or pages, your goal is to make the girhub have updated files but appear as if first commit, ensure no ai attribution, wipe commit and push history, wipe release history and make the 3.5.3 the new release"}

package/embedding-sandbox/specmem/sockets/embedding-death-reason.txt

@@ -0,0 +1,3 @@
+kys
+1769974323.4915226
+No heartbeat from MCP in 90s

package/embedding-sandbox/specmem/sockets/seen-sessions.json

@@ -0,0 +1 @@
+{"b94fecc8-54c6-494a-88ff-0a2898ae9a37":1770239517444}

package/embedding-sandbox/specmem/sockets/session-start.lock

@@ -0,0 +1 @@
+{"sessionId":"b94fecc8-54c6-494a-88ff-0a2898ae9a37","timestamp":1770239517444}

package/embedding-sandbox/specmem/sockets/session-stops.log

@@ -0,0 +1,7 @@
+2026-01-26T10:06:24.973Z | Session stopped: unknown
+2026-01-26T14:49:30.078Z | Session stopped: unknown
+2026-01-30T01:44:41.510Z | Session stopped: unknown
+2026-01-30T01:53:46.929Z | Session stopped: unknown
+2026-02-01T19:29:31.658Z | Session stopped: unknown
+2026-02-01T19:29:53.758Z | Session stopped: unknown
+2026-02-01T19:30:50.851Z | Session stopped: unknown