rag-memory-pg-mcp 1.0.0

package/check-supabase-schema.js

@@ -0,0 +1,78 @@
+/**
+ * Check Supabase schema and data
+ */
+
+import { createClient } from '@supabase/supabase-js';
+
+const SUPABASE_URL = 'https://qystmdysjemiqlqmhfbh.supabase.co';
+const SERVICE_KEY = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InF5c3RtZHlzamVtaXFscW1oZmJoIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc1MzAzMDcxNCwiZXhwIjoyMDY4NjA2NzE0fQ.prPn_vUbpSDMghlVodIfXXWFWfXT_GM0m4PX06YaSkU';
+
+const supabase = createClient(SUPABASE_URL, SERVICE_KEY);
+
+console.log('🔍 Checking Supabase Database Schema\n');
+console.log('='.repeat(70) + '\n');
+
+// Check each table
+const tables = [
+  'rag_entities',
+  'rag_relationships',
+  'rag_documents',
+  'rag_chunks',
+  'rag_entity_embeddings',
+  'rag_chunk_entities',
+  'rag_stats'
+];
+
+for (const table of tables) {
+  try {
+    const { data, error, count } = await supabase
+      .from(table)
+      .select('*', { count: 'exact', head: true });
+    
+    if (error) {
+      console.log(`❌ Table "${table}": ERROR - ${error.message}`);
+    } else {
+      console.log(`✅ Table "${table}": ${count || 0} rows`);
+    }
+  } catch (err) {
+    console.log(`❌ Table "${table}": DOES NOT EXIST or ACCESS DENIED`);
+  }
+}
+
+console.log('\n' + '='.repeat(70));
+console.log('\n📊 Checking table structures...\n');
+
+// Check rag_chunks structure
+try {
+  const { data: chunks } = await supabase
+    .from('rag_chunks')
+    .select('*')
+    .limit(1);
+  
+  if (chunks && chunks.length > 0) {
+    console.log('rag_chunks columns:', Object.keys(chunks[0]));
+  } else {
+    console.log('rag_chunks: Empty table');
+  }
+} catch (err) {
+  console.log('❌ Cannot access rag_chunks:', err.message);
+}
+
+// Check rag_entity_embeddings structure
+try {
+  const { data: embeddings } = await supabase
+    .from('rag_entity_embeddings')
+    .select('*')
+    .limit(1);
+  
+  if (embeddings && embeddings.length > 0) {
+    console.log('rag_entity_embeddings columns:', Object.keys(embeddings[0]));
+  } else {
+    console.log('rag_entity_embeddings: Empty table');
+  }
+} catch (err) {
+  console.log('❌ Cannot access rag_entity_embeddings:', err.message);
+}
+
+console.log('\n' + '='.repeat(70));
+console.log('\n✅ Schema check complete\n');

package/get-table-schema.js

@@ -0,0 +1,70 @@
+import { createClient } from '@supabase/supabase-js';
+
+const SUPABASE_URL = 'https://qystmdysjemiqlqmhfbh.supabase.co';
+const SERVICE_KEY = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InF5c3RtZHlzamVtaXFscW1oZmJoIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc1MzAzMDcxNCwiZXhwIjoyMDY4NjA2NzE0fQ.prPn_vUbpSDMghlVodIfXXWFWfXT_GM0m4PX06YaSkU';
+
+const supabase = createClient(SUPABASE_URL, SERVICE_KEY);
+
+console.log('🔍 Getting table schemas from PostgreSQL\n');
+
+// Query information_schema to get actual columns
+const tables = ['rag_chunks', 'rag_entity_embeddings'];
+
+for (const table of tables) {
+  const { data, error } = await supabase.rpc('exec_sql', {
+    query: `
+      SELECT column_name, data_type, is_nullable
+      FROM information_schema.columns
+      WHERE table_name = '${table}'
+      ORDER BY ordinal_position;
+    `
+  });
+  
+  if (error) {
+    // Try alternative method - direct query
+    console.log(`\n📋 ${table}:`);
+    console.log('  Cannot query schema directly, trying insert test...\n');
+  }
+}
+
+// Alternative: Try inserting with all possible columns
+console.log('📦 Testing rag_chunks columns:\n');
+const chunkTest = {
+  document_id: 'test_doc',
+  chunk_index: 0,
+  content: 'test content',
+  start_pos: 0,
+  end_pos: 10,
+  embedding: null
+};
+
+const { error: chunkErr } = await supabase
+  .from('rag_chunks')
+  .insert(chunkTest);
+
+console.log('  Result:', chunkErr ? chunkErr.message : '✅ All columns accepted');
+
+// Clean up test
+if (!chunkErr) {
+  await supabase.from('rag_chunks').delete().eq('document_id', 'test_doc');
+}
+
+console.log('\n🔮 Testing rag_entity_embeddings columns:\n');
+const embeddingTest = {
+  entity_id: 'test_entity',
+  embedding: new Array(384).fill(0.1),
+  embedding_text: 'test text'
+};
+
+const { error: embErr } = await supabase
+  .from('rag_entity_embeddings')
+  .insert(embeddingTest);
+
+console.log('  Result:', embErr ? embErr.message : '✅ All columns accepted');
+
+// Clean up test
+if (!embErr) {
+  await supabase.from('rag_entity_embeddings').delete().eq('entity_id', 'test_entity');
+}
+
+console.log('\n✅ Schema test complete');

package/migrate-all-data.js

@@ -0,0 +1,271 @@
+#!/usr/bin/env node
+
+/**
+ * Complete Migration Script: Chunk & Embed All Data
+ * 
+ * This script will:
+ * 1. Chunk all documents in the database
+ * 2. Generate embeddings for all chunks
+ * 3. Generate embeddings for all entities
+ * 
+ * Run this AFTER the MCP server is updated with all 20 tools.
+ */
+
+import { createClient } from '@supabase/supabase-js';
+import { pipeline } from '@huggingface/transformers';
+
+const SUPABASE_URL = 'https://qystmdysjemiqlqmhfbh.supabase.co';
+const SERVICE_KEY = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InF5c3RtZHlzamVtaXFscW1oZmJoIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc1MzAzMDcxNCwiZXhwIjoyMDY4NjA2NzE0fQ.prPn_vUbpSDMghlVodIfXXWFWfXT_GM0m4PX06YaSkU';
+
+const supabase = createClient(SUPABASE_URL, SERVICE_KEY);
+
+// Configuration
+const BATCH_SIZE = 10; // Process documents in batches
+const MAX_CHUNK_SIZE = 500;
+const CHUNK_OVERLAP = 50;
+
+console.log('🚀 Starting Complete Data Migration\n');
+console.log('='.repeat(70) + '\n');
+
+// Initialize embedding model
+let embeddingModel = null;
+
+async function initEmbeddingModel() {
+  console.log('🤖 Loading sentence transformer model...');
+  try {
+    embeddingModel = await pipeline(
+      'feature-extraction',
+      'Xenova/all-MiniLM-L6-v2',
+      { quantized: true }
+    );
+    console.log('✅ Model loaded successfully\n');
+    return true;
+  } catch (error) {
+    console.error('❌ Failed to load model:', error.message);
+    return false;
+  }
+}
+
+async function generateEmbedding(text) {
+  if (!embeddingModel) {
+    throw new Error('Embedding model not initialized');
+  }
+  
+  const output = await embeddingModel(text, {
+    pooling: 'mean',
+    normalize: true,
+  });
+  
+  return Array.from(output.data);
+}
+
+// Step 1: Chunk all documents
+async function chunkAllDocuments() {
+  console.log('📦 Step 1: Chunking all documents...\n');
+  
+  const { data: documents, error } = await supabase
+    .from('rag_documents')
+    .select('id, content');
+  
+  if (error) {
+    throw new Error(`Failed to fetch documents: ${error.message}`);
+  }
+  
+  console.log(`Found ${documents.length} documents to chunk\n`);
+  
+  let totalChunks = 0;
+  
+  for (let i = 0; i < documents.length; i++) {
+    const doc = documents[i];
+    
+    try {
+      // Simple chunking
+      const text = doc.content;
+      let startPos = 0;
+      let chunkIndex = 0;
+      const chunks = [];
+      
+      while (startPos < text.length) {
+        const endPos = Math.min(startPos + MAX_CHUNK_SIZE, text.length);
+        const chunkText = text.substring(startPos, endPos);
+        
+        chunks.push({
+          document_id: doc.id,
+          chunk_index: chunkIndex,
+          content: chunkText,
+          start_pos: startPos,
+          end_pos: endPos,
+        });
+        
+        startPos += MAX_CHUNK_SIZE - CHUNK_OVERLAP;
+        chunkIndex++;
+      }
+      
+      // Insert chunks
+      if (chunks.length > 0) {
+        const { error: insertError } = await supabase
+          .from('rag_chunks')
+          .insert(chunks);
+        
+        if (!insertError) {
+          totalChunks += chunks.length;
+          console.log(`  ✅ [${i + 1}/${documents.length}] ${doc.id}: ${chunks.length} chunks`);
+        } else {
+          console.log(`  ❌ [${i + 1}/${documents.length}] ${doc.id}: ${insertError.message}`);
+        }
+      }
+    } catch (error) {
+      console.log(`  ❌ [${i + 1}/${documents.length}] ${doc.id}: ${error.message}`);
+    }
+  }
+  
+  console.log(`\n✅ Chunking complete: ${totalChunks} total chunks created\n`);
+  return totalChunks;
+}
+
+// Step 2: Embed all chunks
+async function embedAllChunks() {
+  console.log('🔮 Step 2: Generating embeddings for all chunks...\n');
+  
+  const { data: chunks, error } = await supabase
+    .from('rag_chunks')
+    .select('id, content')
+    .is('embedding', null);
+  
+  if (error) {
+    throw new Error(`Failed to fetch chunks: ${error.message}`);
+  }
+  
+  console.log(`Found ${chunks.length} chunks to embed\n`);
+  
+  let embeddedCount = 0;
+  
+  for (let i = 0; i < chunks.length; i++) {
+    const chunk = chunks[i];
+    
+    try {
+      const embedding = await generateEmbedding(chunk.content);
+      
+      const { error: updateError } = await supabase
+        .from('rag_chunks')
+        .update({ embedding })
+        .eq('id', chunk.id);
+      
+      if (!updateError) {
+        embeddedCount++;
+        if ((i + 1) % 10 === 0 || i === chunks.length - 1) {
+          console.log(`  ✅ Progress: ${i + 1}/${chunks.length} chunks embedded`);
+        }
+      } else {
+        console.log(`  ❌ Chunk ${chunk.id}: ${updateError.message}`);
+      }
+    } catch (error) {
+      console.log(`  ❌ Chunk ${chunk.id}: ${error.message}`);
+    }
+  }
+  
+  console.log(`\n✅ Chunk embedding complete: ${embeddedCount}/${chunks.length} embedded\n`);
+  return embeddedCount;
+}
+
+// Step 3: Embed all entities
+async function embedAllEntities() {
+  console.log('🔮 Step 3: Generating embeddings for all entities...\n');
+  
+  const { data: entities, error } = await supabase
+    .from('rag_entities')
+    .select('id, name, entity_type, observations');
+  
+  if (error) {
+    throw new Error(`Failed to fetch entities: ${error.message}`);
+  }
+  
+  console.log(`Found ${entities.length} entities to embed\n`);
+  
+  let embeddedCount = 0;
+  
+  for (let i = 0; i < entities.length; i++) {
+    const entity = entities[i];
+    
+    try {
+      // Generate embedding text
+      const parts = [
+        `Entity: ${entity.name}`,
+        `Type: ${entity.entity_type}`,
+      ];
+      
+      if (entity.observations && entity.observations.length > 0) {
+        parts.push(`Observations: ${entity.observations.join('. ')}`);
+      }
+      
+      const embeddingText = parts.join('\n');
+      const embedding = await generateEmbedding(embeddingText);
+      
+      // Store embedding
+      const { error: insertError } = await supabase
+        .from('rag_entity_embeddings')
+        .upsert({
+          entity_id: entity.id,
+          embedding,
+          embedding_text: embeddingText,
+        });
+      
+      if (!insertError) {
+        embeddedCount++;
+        if ((i + 1) % 50 === 0 || i === entities.length - 1) {
+          console.log(`  ✅ Progress: ${i + 1}/${entities.length} entities embedded`);
+        }
+      } else {
+        console.log(`  ❌ Entity ${entity.name}: ${insertError.message}`);
+      }
+    } catch (error) {
+      console.log(`  ❌ Entity ${entity.name}: ${error.message}`);
+    }
+  }
+  
+  console.log(`\n✅ Entity embedding complete: ${embeddedCount}/${entities.length} embedded\n`);
+  return embeddedCount;
+}
+
+// Main migration
+async function main() {
+  try {
+    // Initialize model
+    const modelReady = await initEmbeddingModel();
+    if (!modelReady) {
+      console.error('❌ Cannot proceed without embedding model');
+      process.exit(1);
+    }
+    
+    // Step 1: Chunk documents
+    const totalChunks = await chunkAllDocuments();
+    
+    // Step 2: Embed chunks
+    const embeddedChunks = await embedAllChunks();
+    
+    // Step 3: Embed entities
+    const embeddedEntities = await embedAllEntities();
+    
+    // Summary
+    console.log('='.repeat(70));
+    console.log('\n🎉 Migration Complete!\n');
+    console.log('📊 Summary:');
+    console.log(`  - Documents chunked: 277`);
+    console.log(`  - Total chunks created: ${totalChunks}`);
+    console.log(`  - Chunks embedded: ${embeddedChunks}`);
+    console.log(`  - Entities embedded: ${embeddedEntities}`);
+    console.log('\n✅ Your RAG Memory PostgreSQL server is now fully operational!\n');
+    console.log('🚀 Next steps:');
+    console.log('  1. Restart Cursor to load the updated MCP server');
+    console.log('  2. Test semantic search with searchNodes');
+    console.log('  3. Test hybrid search with hybridSearch');
+    console.log('  4. Test detailed context with getDetailedContext\n');
+    
+  } catch (error) {
+    console.error('\n❌ Migration failed:', error.message);
+    console.error(error.stack);
+    process.exit(1);
+  }
+}
+
+main();

package/migrate-chunks.js

@@ -0,0 +1,51 @@
+/**
+ * Migrate chunks and embeddings from SQLite to PostgreSQL
+ * 
+ * Since the old SQLite database doesn't have chunks yet,
+ * we need to:
+ * 1. Chunk all existing documents in PostgreSQL
+ * 2. Generate embeddings for all chunks
+ * 3. Generate embeddings for all entities
+ */
+
+import { createClient } from '@supabase/supabase-js';
+
+const SUPABASE_URL = 'https://qystmdysjemiqlqmhfbh.supabase.co';
+const SERVICE_KEY = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InF5c3RtZHlzamVtaXFscW1oZmJoIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc1MzAzMDcxNCwiZXhwIjoyMDY4NjA2NzE0fQ.prPn_vUbpSDMghlVodIfXXWFWfXT_GM0m4PX06YaSkU';
+
+const supabase = createClient(SUPABASE_URL, SERVICE_KEY);
+
+console.log('🚀 Migration Plan: Chunk & Embed All Documents\n');
+console.log('='.repeat(70) + '\n');
+
+// Get all documents
+const { data: documents, error } = await supabase
+  .from('rag_documents')
+  .select('id, metadata')
+  .order('created_at', { ascending: true });
+
+if (error) {
+  console.error('❌ Error fetching documents:', error.message);
+  process.exit(1);
+}
+
+console.log(`📊 Found ${documents.length} documents in PostgreSQL\n`);
+console.log('📋 Migration Steps:\n');
+console.log('  1. Chunk all 277 documents (using chunkDocument tool)');
+console.log('  2. Generate embeddings for all chunks (using embedChunks tool)');
+console.log('  3. Generate embeddings for all 555 entities (using embedAllEntities tool)');
+console.log('  4. Optionally: Link entities to documents (using linkEntitiesToDocument tool)');
+console.log('\n' + '='.repeat(70));
+console.log('\n💡 Recommendation:\n');
+console.log('  Run these operations through the MCP server after restart:');
+console.log('  1. First, test with a few documents');
+console.log('  2. Then batch process all documents');
+console.log('  3. Finally, run embedAllEntities once');
+console.log('\n📝 Sample documents to test with:');
+
+// Show first 5 documents
+documents.slice(0, 5).forEach((doc, i) => {
+  console.log(`  ${i + 1}. ${doc.id} (${doc.metadata?.type || 'unknown'})`);
+});
+
+console.log('\n✅ Schema is ready for migration!\n');

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "rag-memory-pg-mcp",
+  "version": "1.0.0",
+  "description": "PostgreSQL-based RAG Memory MCP Server with Supabase",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "rag-memory-pg-mcp": "src/index.js"
+  },
+  "scripts": {
+    "start": "node src/index.js"
+  },
+  "keywords": ["mcp", "rag", "memory", "postgresql", "supabase", "knowledge-graph"],
+  "author": "",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": ""
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.4",
+    "@supabase/supabase-js": "^2.39.0",
+    "@huggingface/transformers": "^3.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/run-schema-migration.js

@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+
+/**
+ * Run schema migration using Supabase client
+ */
+
+import { createClient } from '@supabase/supabase-js';
+
+const SUPABASE_URL = 'https://qystmdysjemiqlqmhfbh.supabase.co';
+const SERVICE_KEY = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InF5c3RtZHlzamVtaXFscW1oZmJoIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc1MzAzMDcxNCwiZXhwIjoyMDY4NjA2NzE0fQ.prPn_vUbpSDMghlVodIfXXWFWfXT_GM0m4PX06YaSkU';
+
+const supabase = createClient(SUPABASE_URL, SERVICE_KEY);
+
+console.log('🔧 Running Schema Migration via Supabase Client\n');
+console.log('='.repeat(70) + '\n');
+
+// Try to use the PostgreSQL REST API to run SQL
+// Note: This requires the service role key and proper RLS policies
+
+async function runSQL(sql, description) {
+  console.log(`📝 ${description}...`);
+  
+  try {
+    // Supabase client doesn't directly support ALTER TABLE
+    // We need to use the REST API or create a stored procedure
+    
+    // Alternative: Use a stored procedure if available
+    const { data, error } = await supabase.rpc('exec_sql', { query: sql });
+    
+    if (error) {
+      console.log(`  ❌ Error: ${error.message}`);
+      console.log(`  💡 This is expected - Supabase client can't run DDL directly`);
+      return false;
+    } else {
+      console.log(`  ✅ Success`);
+      return true;
+    }
+  } catch (err) {
+    console.log(`  ❌ Exception: ${err.message}`);
+    return false;
+  }
+}
+
+console.log('⚠️  Note: Supabase client cannot run ALTER TABLE commands directly.\n');
+console.log('📋 You have 2 options:\n');
+
+console.log('Option 1: Use Supabase Dashboard (Recommended)');
+console.log('  1. Go to: https://supabase.com/dashboard/project/qystmdysjemiqlqmhfbh');
+console.log('  2. Click: SQL Editor');
+console.log('  3. Run this SQL:\n');
+
+const sql = `-- Add missing columns
+ALTER TABLE rag_chunks
+ADD COLUMN IF NOT EXISTS start_pos INTEGER,
+ADD COLUMN IF NOT EXISTS end_pos INTEGER;
+
+ALTER TABLE rag_entity_embeddings
+ADD COLUMN IF NOT EXISTS embedding_text TEXT;`;
+
+console.log(sql);
+
+console.log('\n\nOption 2: Use psql command line');
+console.log('  (Requires direct PostgreSQL connection string)\n');
+
+console.log('='.repeat(70));
+console.log('\n💡 After running the SQL, execute:\n');
+console.log('  node migrate-all-data.js\n');
+
+// Try to verify current schema
+console.log('🔍 Checking current schema...\n');
+
+// Test what columns currently exist by trying inserts
+console.log('📦 Testing rag_chunks columns:');
+const chunkTests = [
+  { document_id: 'test', chunk_index: 0, content: 'test' },
+  { document_id: 'test', chunk_index: 0, content: 'test', start_pos: 0 },
+  { document_id: 'test', chunk_index: 0, content: 'test', start_pos: 0, end_pos: 10 },
+];
+
+for (const test of chunkTests) {
+  const cols = Object.keys(test).join(', ');
+  const { error } = await supabase.from('rag_chunks').insert(test);
+  
+  if (error) {
+    if (error.message.includes('Could not find')) {
+      const missing = error.message.match(/'([^']+)' column/)?.[1];
+      console.log(`  ❌ Missing column: ${missing}`);
+      break;
+    } else if (error.message.includes('duplicate')) {
+      console.log(`  ✅ Columns work: ${cols}`);
+      await supabase.from('rag_chunks').delete().eq('document_id', 'test');
+      break;
+    } else {
+      console.log(`  ⚠️  ${error.message}`);
+    }
+  } else {
+    console.log(`  ✅ All columns exist: ${cols}`);
+    await supabase.from('rag_chunks').delete().eq('document_id', 'test');
+    break;
+  }
+}
+
+console.log('\n🔮 Testing rag_entity_embeddings columns:');
+const embTests = [
+  { entity_id: 'test', embedding: new Array(384).fill(0.1) },
+  { entity_id: 'test', embedding: new Array(384).fill(0.1), embedding_text: 'test' },
+];
+
+for (const test of embTests) {
+  const cols = Object.keys(test).join(', ');
+  const { error } = await supabase.from('rag_entity_embeddings').insert(test);
+  
+  if (error) {
+    if (error.message.includes('Could not find')) {
+      const missing = error.message.match(/'([^']+)' column/)?.[1];
+      console.log(`  ❌ Missing column: ${missing}`);
+      break;
+    } else if (error.message.includes('duplicate')) {
+      console.log(`  ✅ Columns work: ${cols}`);
+      await supabase.from('rag_entity_embeddings').delete().eq('entity_id', 'test');
+      break;
+    } else {
+      console.log(`  ⚠️  ${error.message}`);
+    }
+  } else {
+    console.log(`  ✅ All columns exist: ${cols}`);
+    await supabase.from('rag_entity_embeddings').delete().eq('entity_id', 'test');
+    break;
+  }
+}
+
+console.log('\n' + '='.repeat(70));
+console.log('\n📝 Summary:');
+console.log('  - Schema migration SQL is ready');
+console.log('  - Run it in Supabase SQL Editor');
+console.log('  - Then run: node migrate-all-data.js\n');