squish-memory 1.1.5 → 1.2.0

package/core/embeddings/transformers-local.ts

@@ -0,0 +1,261 @@
+/**
+ * Transformers.js Local Embedding Provider
+ *
+ * Uses ONNX-based transformer models for high-quality local embeddings.
+ * Supports Hugging Face ONNX embedding models.
+ *
+ * Usage:
+ *   Set SQUISH_EMBEDDINGS_PROVIDER=transformers
+ *   Required: SQUISH_LOCAL_MODEL=<huggingface-onnx-model>
+ *
+ * Download models automatically on first use. Models cached in HuggingFace cache directory.
+ */
+
+import { pipeline, Pipeline } from '@huggingface/transformers';
+import { logger } from '../logger.js';
+import { config } from '../../config.js';
+
+export interface TransformersLocalConfig {
+  model: string;
+  device: 'cpu' | 'webgpu';
+  dtype: 'q8' | 'q4' | 'f16' | 'f32';
+}
+
+const DEFAULT_CONFIG: TransformersLocalConfig = {
+  model: '',
+  device: 'cpu',
+  dtype: 'q8', // Quantized for smaller size + faster loading
+};
+
+// Singleton pipeline instance
+let embeddingPipeline: Pipeline | null = null;
+let isLoading = false;
+let loadPromise: Promise<Pipeline | null> | null = null;
+
+/**
+ * Get or initialize the embedding pipeline (lazy loading)
+ */
+async function getPipeline(): Promise<Pipeline | null> {
+  // Return existing pipeline
+  if (embeddingPipeline) {
+    return embeddingPipeline;
+  }
+
+  // Already loading - wait for it
+  if (isLoading && loadPromise) {
+    return loadPromise;
+  }
+
+  // Start loading
+  isLoading = true;
+  const model = config.transformersLocalModel;
+  if (!model) {
+    isLoading = false;
+    throw new Error('Transformers provider requires SQUISH_LOCAL_MODEL to be set');
+  }
+
+  logger.info(`Loading transformers local model: ${model}`);
+
+  loadPromise = (async () => {
+    try {
+      embeddingPipeline = await pipeline(
+        'feature-extraction',
+        model as any,
+        {
+          device: DEFAULT_CONFIG.device as any,
+          dtype: DEFAULT_CONFIG.dtype as any,
+        }
+      );
+
+      logger.info(`Transformers local model loaded: ${model}`);
+      return embeddingPipeline;
+    } catch (error) {
+      logger.error(`Failed to load transformers model: ${error}`);
+      embeddingPipeline = null;
+      throw error;
+    } finally {
+      isLoading = false;
+      loadPromise = null;
+    }
+  })();
+
+  return loadPromise;
+}
+
+/**
+ * Check if pipeline is loaded
+ */
+export function isReady(): boolean {
+  return embeddingPipeline !== null;
+}
+
+/**
+ * Get embedding dimension for current model
+ */
+export function getEmbeddingDimension(): number {
+  return 0;
+}
+
+/**
+ * Generate embedding for a single text input
+ * Uses mean pooling + L2 normalization
+ */
+export async function getEmbedding(text: string): Promise<number[] | null> {
+  if (!text || typeof text !== 'string') {
+    return null;
+  }
+
+  try {
+    const pipeline = await getPipeline();
+    if (!pipeline) {
+      return null;
+    }
+
+    // Truncate to max sequence length (256 tokens)
+    const truncatedText = text.slice(0, 512);
+
+    // Run feature extraction with pooling
+    const output = await pipeline(truncatedText, {
+      pooling: 'mean',
+      normalize: true,
+    });
+
+    // Convert tensor to array
+    const embedding = Array.from(output.data);
+
+    return embedding;
+  } catch (error) {
+    logger.warn(`Transformers embedding error: ${error}`);
+    return null;
+  }
+}
+
+/**
+ * Generate embeddings for multiple texts in batch
+ * Processes efficiently with batching
+ */
+export async function getBatchEmbeddings(
+  texts: string[],
+  batchSize: number = 8
+): Promise<Array<number[] | null>> {
+  if (texts.length === 0) {
+    return [];
+  }
+
+  const results: Array<number[] | null> = new Array(texts.length).fill(null);
+
+  // Process in batches
+  for (let i = 0; i < texts.length; i += batchSize) {
+    const batchEnd = Math.min(i + batchSize, texts.length);
+    const batch = texts.slice(i, batchEnd);
+    const indices = [...Array(batch.length).keys()].map(j => i + j);
+
+    try {
+      const pipeline = await getPipeline();
+      if (!pipeline) {
+        continue;
+      }
+
+      // Truncate each text
+      const truncatedBatch = batch.map(t => t.slice(0, 512));
+
+      // Process batch
+      const outputs = await Promise.all(
+        truncatedBatch.map(text =>
+          pipeline(text, { pooling: 'mean', normalize: true })
+        )
+      );
+
+      // Store results
+      for (let j = 0; j < outputs.length; j++) {
+        results[indices[j]] = Array.from(outputs[j].data);
+      }
+    } catch (error) {
+      logger.warn(`Transformers batch embedding error: ${error}`);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Check health of the transformers provider
+ */
+export async function checkHealth(): Promise<{
+  available: boolean;
+  latencyMs?: number;
+  error?: string;
+  model?: string;
+  dimension?: number;
+}> {
+  const model = config.transformersLocalModel;
+  if (!model) {
+    return {
+      available: false,
+      error: 'SQUISH_LOCAL_MODEL is not configured',
+    };
+  }
+
+  // Check if library is available
+  try {
+    await import('@huggingface/transformers');
+  } catch (error) {
+    return {
+      available: false,
+      error: '@huggingface/transformers not installed',
+    };
+  }
+
+  // Try quick embedding
+  const start = Date.now();
+  try {
+    const embedding = await getEmbedding('health check test');
+    const latency = Date.now() - start;
+
+    return {
+      available: embedding !== null && embedding.length > 0,
+      latencyMs: latency,
+      model,
+      dimension: embedding?.length || 0,
+    };
+  } catch (error) {
+    return {
+      available: false,
+      error: (error as Error).message,
+      model,
+    };
+  }
+}
+
+/**
+ * Unload the pipeline (for testing or memory management)
+ */
+export async function unload(): Promise<void> {
+  if (embeddingPipeline) {
+    // Pipeline doesn't have explicit cleanup, just release reference
+    embeddingPipeline = null;
+    logger.info('Transformers pipeline unloaded');
+  }
+}
+
+/**
+ * Warm up the model with a test input
+ */
+export async function warmup(): Promise<boolean> {
+  try {
+    const result = await getEmbedding('warmup test');
+    return result !== null && result.length > 0;
+  } catch {
+    return false;
+  }
+}
+
+export default {
+  isReady,
+  getEmbeddingDimension,
+  getEmbedding,
+  getBatchEmbeddings,
+  checkHealth,
+  unload,
+  warmup,
+};

package/core/embeddings.js

@@ -0,0 +1,4 @@
+// Re-export from embeddings directory
+export * from './embeddings/embeddings';
+export * from './embeddings/google-multimodal';
+export * from './embeddings/local-embeddings';

package/core/error-handling.ts

@@ -0,0 +1,206 @@
+/**
+ * Unified Error Handling System for Squish
+ * Provides standardized error handling across CLI, Web API, MCP server, and algorithm handlers
+ */
+
+import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
+import { isDatabaseUnavailableError } from './lib/utils.js';
+
+export { isDatabaseUnavailableError };
+
+// Database operation wrapper
+
+/**
+ * Wraps database operations with standardized error handling
+ */
+export async function withDbErrorHandling<T>(
+  operation: () => Promise<T>,
+  context: string
+): Promise<T> {
+  try {
+    return await operation();
+  } catch (dbError: any) {
+    if (isDatabaseUnavailableError(dbError)) {
+      throw new Error(`Database unavailable: ${context}. Please check your database connection.`);
+    }
+    throw dbError;
+  }
+}
+
+// API Response Formatters
+
+/**
+ * Formats errors for Web API responses
+ */
+export function formatApiError(
+  error: any,
+  defaultStatus: number = 500
+): { status: number; response: { status: string; message: string } } {
+  let message: string;
+  
+  if (typeof error === 'string') {
+    message = error;
+  } else if (error instanceof Error) {
+    message = error.message;
+  } else if (error && typeof error === 'object' && 'message' in error) {
+    message = String(error.message);
+  } else if (error && typeof error === 'object') {
+    message = 'Unknown error';
+  } else {
+    message = 'Unknown error';
+  }
+  
+  // Check if it's a database unavailable error
+  if (isDatabaseUnavailableError(error)) {
+    return {
+      status: 503,
+      response: {
+        status: 'error',
+        message: `Service unavailable: ${message}`
+      }
+    };
+  }
+  
+  // Check for custom error classes with status hints
+  if (error.status) {
+    return {
+      status: error.status,
+      response: {
+        status: 'error',
+        message
+      }
+    };
+  }
+  
+  return {
+    status: defaultStatus,
+    response: {
+      status: 'error',
+      message
+    }
+  };
+}
+
+// MCP Error Formatter
+
+/**
+ * Formats errors for MCP tool responses
+ */
+export function formatMcpError(
+  error: any,
+  context?: string
+): never {
+  const message = error?.message || error?.toString() || 'Unknown error';
+  const fullMessage = context ? `${context}: ${message}` : message;
+  
+  // Determine appropriate error code
+  let code: ErrorCode;
+  
+  if (isDatabaseUnavailableError(error)) {
+    code = ErrorCode.InternalError; // -32603
+  } else if (error instanceof ValidationError) {
+    code = ErrorCode.InvalidParams; // -32602
+  } else {
+    code = ErrorCode.InternalError;
+  }
+  
+  throw new McpError(code, fullMessage);
+}
+
+// CLI Error Formatter
+
+/**
+ * Formats errors for CLI commands
+ */
+export function formatCliError(
+  error: any,
+  exitCode: number = 1
+): never {
+  let message: string;
+  
+  if (typeof error === 'string') {
+    message = error;
+  } else if (error instanceof Error) {
+    message = error.message;
+  } else if (error && typeof error === 'object' && 'message' in error) {
+    message = String(error.message);
+  } else {
+    message = 'Unknown error';
+  }
+  
+  // Output JSON to stdout for CLI consumers
+  console.log(JSON.stringify({ ok: false, error: message }, null, 2));
+  process.exit(exitCode);
+}
+
+// Algorithm Handler Formatters
+
+/**
+ * Formats errors for algorithm handlers
+ */
+export function formatAlgorithmError(
+  error: any,
+  context?: string
+): { ok: false; error: string } {
+  let message: string;
+  
+  if (typeof error === 'string') {
+    message = error;
+  } else if (error instanceof Error) {
+    message = error.message;
+  } else if (error && typeof error === 'object' && 'message' in error) {
+    message = String(error.message);
+  } else {
+    message = 'Unknown error';
+  }
+  
+  const fullMessage = context ? `${context}: ${message}` : message;
+  return { ok: false, error: fullMessage };
+}
+
+/**
+ * Formats successful responses for algorithm handlers.
+ * 
+ * @param data - The success data to return
+ * @param message - Optional success message
+ * @returns Object with ok: true, optional message, and data
+ */
+export function formatAlgorithmSuccess<T>(
+  data: T,
+  message?: string
+): { ok: true; data: T; message?: string } {
+  const result: { ok: true; data: T; message?: string } = { ok: true, data };
+  if (message) {
+    result.message = message;
+  }
+  return result;
+}
+
+// Custom Error Classes
+
+/**
+ * Error thrown when validation fails
+ */
+export class ValidationError extends Error {
+  constructor(
+    message: string,
+    public field?: string
+  ) {
+    super(message);
+    this.name = 'ValidationError';
+  }
+}
+
+/**
+ * Error thrown when a requested resource is not found.
+ * Used for missing memories, projects, etc.
+ */
+export class NotFoundError extends Error {
+  constructor(
+    message: string,
+    public resource?: string
+  ) {
+    super(message);
+    this.name = 'NotFoundError';
+  }
+}

package/core/external

@@ -0,0 +1,219 @@
+import { config } from '../config.js';
+import { logger } from './logger.js';
+import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from 'fs';
+import { join, dirname } from 'path';
+
+const QMD_DIR = 'qmd';
+
+interface QMDMemory {
+  id: string;
+  content: string;
+  type: string;
+  tags: string[];
+  createdAt: string;
+  updatedAt: string;
+  tier: string;
+  projectPath?: string;
+}
+
+export function getQMDDir(projectPath?: string): string {
+  const project = projectPath || config.dataDir;
+  return join(project, QMD_DIR);
+}
+
+export function ensureQMDDir(projectPath?: string): string {
+  const dir = getQMDDir(projectPath);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+    logger.info(`[QMD] Created QMD directory: ${dir}`);
+  }
+  return dir;
+}
+
+function parseFrontMatter(content: string): { metadata: Record<string, any>; body: string } {
+  const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (!match) {
+    return { metadata: {}, body: content };
+  }
+  
+  const yamlContent = match[1];
+  const body = match[2];
+  
+  const metadata: Record<string, any> = {};
+  const lines = yamlContent.split('\n');
+  let currentKey = '';
+  let currentArray: string[] = [];
+  
+  for (const line of lines) {
+    const keyMatch = line.match(/^(\w+):\s*(.*)$/);
+    if (keyMatch) {
+      if (currentKey) {
+        if (currentArray.length > 0) {
+          metadata[currentKey] = currentArray;
+          currentArray = [];
+        } else {
+          metadata[currentKey] = currentKey;
+        }
+      }
+      currentKey = keyMatch[1];
+      const value = keyMatch[2].trim();
+      
+      if (value.startsWith('[') && value.endsWith(']')) {
+        currentArray = value.slice(1, -1).split(',').map(s => s.trim().replace(/['"]/g, ''));
+      } else if (value) {
+        metadata[currentKey] = value.replace(/['"]/g, '');
+      }
+    } else if (line.trim().startsWith('-')) {
+      currentArray.push(line.trim().slice(1).trim().replace(/['"]/g, ''));
+    }
+  }
+  
+  if (currentKey) {
+    if (currentArray.length > 0) {
+      metadata[currentKey] = currentArray;
+    }
+  }
+  
+  return { metadata, body };
+}
+
+function serializeMemory(memory: QMDMemory): string {
+  const lines = ['---'];
+  lines.push(`id: ${memory.id}`);
+  lines.push(`type: ${memory.type}`);
+  if (memory.tags && memory.tags.length > 0) {
+    lines.push(`tags: [${memory.tags.join(', ')}]`);
+  }
+  lines.push(`tier: ${memory.tier}`);
+  lines.push(`createdAt: ${memory.createdAt}`);
+  lines.push(`updatedAt: ${memory.updatedAt}`);
+  if (memory.projectPath) {
+    lines.push(`projectPath: ${memory.projectPath}`);
+  }
+  lines.push('---');
+  lines.push('');
+  lines.push(memory.content);
+  return lines.join('\n');
+}
+
+export async function writeMemory(memory: QMDMemory): Promise<void> {
+  const dir = ensureQMDDir(memory.projectPath);
+  const filePath = join(dir, `${memory.id}.md`);
+  const content = serializeMemory(memory);
+  writeFileSync(filePath, content, 'utf-8');
+  logger.debug(`[QMD] Wrote memory to ${filePath}`);
+}
+
+export async function readMemory(memoryId: string, projectPath?: string): Promise<QMDMemory | null> {
+  const dir = getQMDDir(projectPath);
+  const filePath = join(dir, `${memoryId}.md`);
+  
+  if (!existsSync(filePath)) {
+    return null;
+  }
+  
+  try {
+    const content = readFileSync(filePath, 'utf-8');
+    const { metadata, body } = parseFrontMatter(content);
+    
+    return {
+      id: memoryId,
+      content: body,
+      type: metadata.type || 'observation',
+      tags: metadata.tags || [],
+      tier: metadata.tier || 'hot',
+      createdAt: metadata.createdAt || new Date().toISOString(),
+      updatedAt: metadata.updatedAt || new Date().toISOString(),
+      projectPath: metadata.projectPath,
+    };
+  } catch (error) {
+    logger.error(`[QMD] Failed to read memory ${memoryId}:`, error);
+    return null;
+  }
+}
+
+export async function deleteMemory(memoryId: string, projectPath?: string): Promise<boolean> {
+  const dir = getQMDDir(projectPath);
+  const filePath = join(dir, `${memoryId}.md`);
+  
+  if (!existsSync(filePath)) {
+    return false;
+  }
+  
+  try {
+    unlinkSync(filePath);
+    logger.debug(`[QMD] Deleted memory ${memoryId}`);
+    return true;
+  } catch (error) {
+    logger.error(`[QMD] Failed to delete memory ${memoryId}:`, error);
+    return false;
+  }
+}
+
+export async function searchMemories(
+  query: string,
+  projectPath?: string,
+  options?: { limit?: number; tier?: string }
+): Promise<QMDMemory[]> {
+  const dir = getQMDDir(projectPath);
+  const { limit = 50, tier } = options || {};
+  
+  if (!existsSync(dir)) {
+    return [];
+  }
+  
+  const files = readdirSync(dir).filter(f => f.endsWith('.md'));
+  const results: QMDMemory[] = [];
+  const queryLower = query.toLowerCase();
+  
+  for (const file of files) {
+    try {
+      const content = readFileSync(join(dir, file), 'utf-8');
+      const { metadata, body } = parseFrontMatter(content);
+      
+      if (tier && metadata.tier !== tier) {
+        continue;
+      }
+      
+      const searchText = `${metadata.type || ''} ${metadata.tags?.join(' ') || ''} ${body}`.toLowerCase();
+      
+      if (searchText.includes(queryLower)) {
+        results.push({
+          id: file.replace('.md', ''),
+          content: body,
+          type: metadata.type || 'observation',
+          tags: metadata.tags || [],
+          tier: metadata.tier || 'hot',
+          createdAt: metadata.createdAt || '',
+          updatedAt: metadata.updatedAt || '',
+          projectPath: metadata.projectPath,
+        });
+      }
+    } catch {
+      // Skip unreadable files
+    }
+    
+    if (results.length >= limit) break;
+  }
+  
+  return results.slice(0, limit);
+}
+
+export async function getHotMemories(projectPath?: string, limit = 100): Promise<QMDMemory[]> {
+  return searchMemories('', projectPath, { limit, tier: 'hot' });
+}
+
+export async function updateMemoryTier(memoryId: string, newTier: string, projectPath?: string): Promise<boolean> {
+  const memory = await readMemory(memoryId, projectPath);
+  if (!memory) return false;
+  
+  memory.tier = newTier;
+  memory.updatedAt = new Date().toISOString();
+  
+  await writeMemory(memory);
+  return true;
+}
+
+export function isQMDEnabled(): boolean {
+  return config.qmdEnabled !== false;
+}