s3db.js 11.3.2 → 12.0.0

package/mcp/entrypoint.js

@@ -47,6 +47,35 @@ class S3dbMCPServer {
     this.server.setRequestHandler(ListToolsRequestSchema, async () => {
       return {
         tools: [
+          // 📖 DOCUMENTATION TOOLS (for AI agents)
+          {
+            name: 's3dbQueryDocs',
+            description: 'Search s3db.js documentation to answer questions about features, plugins, best practices, and usage. Use this tool to help AI agents understand how to use s3db.js effectively.',
+            inputSchema: {
+              type: 'object',
+              properties: {
+                query: {
+                  type: 'string',
+                  description: 'Natural language question about s3db.js (e.g., "How do I use GeoPlugin?", "What is the best caching strategy?", "How do partitions work?")'
+                },
+                maxResults: {
+                  type: 'number',
+                  description: 'Maximum number of documentation files to return',
+                  default: 5
+                }
+              },
+              required: ['query']
+            }
+          },
+          {
+            name: 's3dbListTopics',
+            description: 'List all available documentation topics and their categories',
+            inputSchema: {
+              type: 'object',
+              properties: {},
+              required: []
+            }
+          },
           {
             name: 'dbConnect',
             description: 'Connect to an S3DB database with automatic costs tracking and configurable cache (memory or filesystem)',
@@ -877,6 +906,15 @@ class S3dbMCPServer {
         let result;
 
         switch (name) {
+          // Documentation tools
+          case 's3dbQueryDocs':
+            result = await this.handleS3dbQueryDocs(args);
+            break;
+
+          case 's3dbListTopics':
+            result = await this.handleS3dbListTopics(args);
+            break;
+
           case 'dbConnect':
             result = await this.handleDbConnect(args);
             break;
@@ -1135,6 +1173,26 @@ class S3dbMCPServer {
     });
   }
 
+  // 📖 DOCUMENTATION TOOLS HANDLERS
+
+  async handleS3dbQueryDocs(args) {
+    const { query, maxResults = 5 } = args;
+
+    // Import the documentation handler dynamically
+    const { createDocumentationHandlers } = await import('./tools/documentation.js');
+    const handlers = createDocumentationHandlers(this);
+
+    return await handlers.s3dbQueryDocs(args);
+  }
+
+  async handleS3dbListTopics(args) {
+    // Import the documentation handler dynamically
+    const { createDocumentationHandlers } = await import('./tools/documentation.js');
+    const handlers = createDocumentationHandlers(this);
+
+    return await handlers.s3dbListTopics(args);
+  }
+
   // Database connection handlers
   async handleDbConnect(args) {
     const { 

package/mcp/tools/documentation.js

@@ -0,0 +1,434 @@
+/**
+ * Documentation Tools
+ * Provides intelligent documentation retrieval for AI agents
+ */
+
+import { readFileSync, readdirSync, statSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const PROJECT_ROOT = join(__dirname, '../..');
+
+/**
+ * Documentation index - maps topics to file paths and sections
+ */
+const DOCUMENTATION_INDEX = {
+  // Core concepts
+  'getting started': ['README.md', 'docs/schema.md'],
+  'installation': ['README.md'],
+  'connection': ['docs/client.md', 'CLAUDE.md'],
+  'database': ['CLAUDE.md', 'docs/schema.md'],
+  'resource': ['CLAUDE.md', 'docs/schema.md'],
+  'schema': ['docs/schema.md', 'CLAUDE.md'],
+  'validation': ['docs/schema.md', 'CLAUDE.md'],
+
+  // Data operations
+  'insert': ['CLAUDE.md'],
+  'update': ['CLAUDE.md'],
+  'delete': ['CLAUDE.md'],
+  'query': ['CLAUDE.md'],
+  'list': ['CLAUDE.md'],
+  'get': ['CLAUDE.md'],
+  'count': ['CLAUDE.md'],
+  'crud': ['CLAUDE.md'],
+
+  // Partitioning
+  'partition': ['CLAUDE.md', 'docs/benchmarks/partitions.md'],
+  'partitioning': ['CLAUDE.md', 'docs/benchmarks/partitions.md'],
+  'orphaned partition': ['CLAUDE.md'],
+  'partition migration': ['CLAUDE.md', 'docs/mcp.md'],
+
+  // Plugins
+  'plugin': ['docs/plugins/README.md', 'CLAUDE.md'],
+  'cache': ['docs/plugins/cache.md', 'CLAUDE.md'],
+  'caching': ['docs/plugins/cache.md', 'CLAUDE.md'],
+  'audit': ['docs/plugins/audit.md', 'CLAUDE.md'],
+  'replicator': ['docs/plugins/replicator.md', 'CLAUDE.md'],
+  'backup': ['docs/plugins/backup.md', 'CLAUDE.md'],
+  'geo': ['docs/plugins/geo.md', 'CLAUDE.md'],
+  'geospatial': ['docs/plugins/geo.md', 'CLAUDE.md'],
+  'location': ['docs/plugins/geo.md'],
+  'metrics': ['docs/plugins/metrics.md', 'CLAUDE.md'],
+  'costs': ['docs/plugins/costs.md', 'CLAUDE.md'],
+  'eventual consistency': ['docs/plugins/eventual-consistency.md', 'CLAUDE.md'],
+  'fulltext': ['docs/plugins/fulltext.md'],
+  'search': ['docs/plugins/fulltext.md'],
+  'queue': ['docs/plugins/queue-consumer.md', 'docs/plugins/s3-queue.md'],
+
+  // Performance & Optimization
+  'performance': ['CLAUDE.md', 'docs/benchmarks/README.md'],
+  'optimization': ['CLAUDE.md', 'docs/benchmarks/README.md'],
+  'benchmark': ['docs/benchmarks/README.md'],
+  'compression': ['CLAUDE.md'],
+  'encoding': ['CLAUDE.md', 'docs/benchmarks/smart-encoding.md'],
+
+  // MCP specific
+  'mcp': ['docs/mcp.md'],
+  'model context protocol': ['docs/mcp.md'],
+  'ai agent': ['docs/mcp.md'],
+  'claude desktop': ['docs/mcp.md'],
+
+  // Advanced features
+  'encryption': ['CLAUDE.md'],
+  'secret': ['CLAUDE.md'],
+  'vector': ['docs/plugins/vector.md'],
+  'embedding': ['CLAUDE.md', 'docs/plugins/vector.md'],
+  'versioning': ['CLAUDE.md'],
+  'hooks': ['CLAUDE.md'],
+  'behavior': ['CLAUDE.md'],
+  'metadata': ['CLAUDE.md'],
+
+  // Troubleshooting
+  'error': ['CLAUDE.md', 'docs/mcp.md'],
+  'troubleshooting': ['docs/mcp.md'],
+  'recovery': ['CLAUDE.md']
+};
+
+/**
+ * Get all markdown files in docs directory
+ */
+function getAllDocFiles() {
+  const files = [];
+
+  function walkDir(dir) {
+    try {
+      const items = readdirSync(dir);
+      for (const item of items) {
+        const fullPath = join(dir, item);
+        try {
+          const stat = statSync(fullPath);
+          if (stat.isDirectory()) {
+            walkDir(fullPath);
+          } else if (item.endsWith('.md')) {
+            const relativePath = fullPath.replace(PROJECT_ROOT + '/', '');
+            files.push(relativePath);
+          }
+        } catch (err) {
+          // Skip files we can't access
+          continue;
+        }
+      }
+    } catch (err) {
+      // Skip directories we can't access
+      return;
+    }
+  }
+
+  walkDir(join(PROJECT_ROOT, 'docs'));
+
+  // Add root-level important docs
+  try {
+    ['README.md', 'CLAUDE.md'].forEach(file => {
+      const fullPath = join(PROJECT_ROOT, file);
+      try {
+        if (statSync(fullPath).isFile()) {
+          files.push(file);
+        }
+      } catch (err) {
+        // Skip if file doesn't exist
+      }
+    });
+  } catch (err) {
+    // Skip if error accessing root files
+  }
+
+  return files;
+}
+
+/**
+ * Search for query terms in documentation
+ */
+function searchDocumentation(query) {
+  const queryLower = query.toLowerCase();
+  const queryTerms = queryLower.split(/\s+/).filter(t => t.length > 2);
+
+  // Find relevant files from index
+  const relevantFiles = new Set();
+
+  // Check index first
+  for (const [topic, files] of Object.entries(DOCUMENTATION_INDEX)) {
+    if (queryLower.includes(topic) || queryTerms.some(term => topic.includes(term))) {
+      files.forEach(f => relevantFiles.add(f));
+    }
+  }
+
+  // If no matches in index, search all docs
+  if (relevantFiles.size === 0) {
+    getAllDocFiles().forEach(f => relevantFiles.add(f));
+  }
+
+  const results = [];
+
+  for (const filePath of relevantFiles) {
+    try {
+      const fullPath = join(PROJECT_ROOT, filePath);
+      const content = readFileSync(fullPath, 'utf-8');
+
+      // Calculate relevance score
+      let score = 0;
+      const contentLower = content.toLowerCase();
+
+      // Exact query match
+      if (contentLower.includes(queryLower)) {
+        score += 100;
+      }
+
+      // Individual term matches
+      queryTerms.forEach(term => {
+        const regex = new RegExp(`\\b${term}\\b`, 'gi');
+        const matches = content.match(regex);
+        if (matches) {
+          score += matches.length * 10;
+        }
+      });
+
+      // Boost for certain file types
+      if (filePath === 'CLAUDE.md') score *= 1.5;
+      if (filePath.includes('plugins/')) score *= 1.2;
+
+      if (score > 0) {
+        // Extract relevant sections
+        const sections = extractRelevantSections(content, queryTerms, filePath);
+
+        results.push({
+          file: filePath,
+          score,
+          sections
+        });
+      }
+    } catch (error) {
+      // Skip files that can't be read
+      continue;
+    }
+  }
+
+  // Sort by relevance
+  results.sort((a, b) => b.score - a.score);
+
+  return results;
+}
+
+/**
+ * Extract relevant sections from markdown content
+ */
+function extractRelevantSections(content, queryTerms, filePath) {
+  const lines = content.split('\n');
+  const sections = [];
+  let currentSection = null;
+  let currentContent = [];
+  let sectionScore = 0;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+
+    // Detect section headers
+    if (line.match(/^#{1,4}\s/)) {
+      // Save previous section if it has content
+      if (currentSection && currentContent.length > 0 && sectionScore > 0) {
+        sections.push({
+          header: currentSection,
+          content: currentContent.join('\n').trim(),
+          score: sectionScore
+        });
+      }
+
+      // Start new section
+      currentSection = line;
+      currentContent = [line];
+      sectionScore = 0;
+
+      // Score the header
+      const headerLower = line.toLowerCase();
+      queryTerms.forEach(term => {
+        if (headerLower.includes(term)) {
+          sectionScore += 50;
+        }
+      });
+    } else if (currentSection) {
+      currentContent.push(line);
+
+      // Score the content
+      const lineLower = line.toLowerCase();
+      queryTerms.forEach(term => {
+        if (lineLower.includes(term)) {
+          sectionScore += 5;
+        }
+      });
+    }
+  }
+
+  // Save last section
+  if (currentSection && currentContent.length > 0 && sectionScore > 0) {
+    sections.push({
+      header: currentSection,
+      content: currentContent.join('\n').trim(),
+      score: sectionScore
+    });
+  }
+
+  // Sort sections by score and limit
+  sections.sort((a, b) => b.score - a.score);
+
+  // Return top sections (limit to avoid huge responses)
+  return sections.slice(0, 3);
+}
+
+/**
+ * Format search results for display
+ */
+function formatResults(results, query, maxResults = 5) {
+  if (results.length === 0) {
+    return {
+      query,
+      found: false,
+      message: 'No documentation found for this query. Try rephrasing or asking about: plugins, cache, partitions, schema, CRUD operations, or MCP integration.',
+      suggestions: [
+        'How do I use the CachePlugin?',
+        'What are partitions and when should I use them?',
+        'How do I create a resource with schema validation?',
+        'How does the MCP server work?',
+        'What plugins are available?'
+      ]
+    };
+  }
+
+  const topResults = results.slice(0, maxResults);
+
+  const formatted = {
+    query,
+    found: true,
+    resultCount: results.length,
+    showing: topResults.length,
+    results: []
+  };
+
+  for (const result of topResults) {
+    const formattedResult = {
+      file: result.file,
+      relevanceScore: result.score,
+      sections: []
+    };
+
+    for (const section of result.sections) {
+      // Limit section content length
+      let content = section.content;
+      if (content.length > 1500) {
+        content = content.substring(0, 1500) + '\n\n... (truncated)';
+      }
+
+      formattedResult.sections.push({
+        header: section.header,
+        content: content
+      });
+    }
+
+    formatted.results.push(formattedResult);
+  }
+
+  return formatted;
+}
+
+export const documentationTools = [
+  {
+    name: 's3dbQueryDocs',
+    description: 'Search s3db.js documentation to answer questions about features, plugins, best practices, and usage. Use this tool to help AI agents understand how to use s3db.js effectively.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        query: {
+          type: 'string',
+          description: 'Natural language question about s3db.js (e.g., "How do I use GeoPlugin?", "What is the best caching strategy?", "How do partitions work?")'
+        },
+        maxResults: {
+          type: 'number',
+          description: 'Maximum number of documentation files to return',
+          default: 5
+        }
+      },
+      required: ['query']
+    }
+  },
+  {
+    name: 's3dbListTopics',
+    description: 'List all available documentation topics and their categories',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+      required: []
+    }
+  }
+];
+
+export function createDocumentationHandlers(server) {
+  return {
+    async s3dbQueryDocs(args) {
+      const { query, maxResults = 5 } = args;
+
+      try {
+        const results = searchDocumentation(query);
+        const formatted = formatResults(results, query, maxResults);
+
+        return {
+          success: true,
+          ...formatted
+        };
+      } catch (error) {
+        return {
+          success: false,
+          query,
+          error: error.message,
+          suggestion: 'Try rephrasing your question or use s3dbListTopics to see available topics'
+        };
+      }
+    },
+
+    async s3dbListTopics(args) {
+      const topics = {};
+
+      // Organize by category
+      topics.core = ['getting started', 'installation', 'connection', 'database', 'resource', 'schema', 'validation'];
+      topics.operations = ['insert', 'update', 'delete', 'query', 'list', 'get', 'count', 'crud'];
+      topics.partitioning = ['partition', 'partitioning', 'orphaned partition', 'partition migration'];
+      topics.plugins = ['plugin', 'cache', 'audit', 'replicator', 'backup', 'geo', 'metrics', 'costs', 'eventual consistency', 'fulltext', 'queue'];
+      topics.performance = ['performance', 'optimization', 'benchmark', 'compression', 'encoding'];
+      topics.mcp = ['mcp', 'model context protocol', 'ai agent', 'claude desktop'];
+      topics.advanced = ['encryption', 'secret', 'vector', 'embedding', 'versioning', 'hooks', 'behavior', 'metadata'];
+      topics.troubleshooting = ['error', 'troubleshooting', 'recovery'];
+
+      const allFiles = getAllDocFiles();
+
+      return {
+        success: true,
+        message: 'Use s3dbQueryDocs with any of these topics to get detailed documentation',
+        categories: topics,
+        availableFiles: allFiles,
+        totalTopics: Object.keys(DOCUMENTATION_INDEX).length,
+        totalFiles: allFiles.length,
+        examples: [
+          {
+            query: 'How do I use the CachePlugin?',
+            description: 'Learn about caching strategies and configuration'
+          },
+          {
+            query: 'What are partitions?',
+            description: 'Understand partitioning for performance optimization'
+          },
+          {
+            query: 'How do I handle orphaned partitions?',
+            description: 'Recovery workflow for partition issues'
+          },
+          {
+            query: 'What plugins are available?',
+            description: 'Complete list of available plugins and their purposes'
+          },
+          {
+            query: 'How does MCP integration work?',
+            description: 'Setting up and using the MCP server with AI agents'
+          }
+        ]
+      };
+    }
+  };
+}

package/mcp/tools/index.js

@@ -12,12 +12,14 @@ import { partitionTools, createPartitionHandlers } from './partitions.js';
 import { bulkTools, createBulkHandlers } from './bulk.js';
 import { exportImportTools, createExportImportHandlers } from './export-import.js';
 import { statsTools, createStatsHandlers } from './stats.js';
+import { documentationTools, createDocumentationHandlers } from './documentation.js';
 
 /**
  * Get all tool definitions
  */
 export function getAllTools() {
   return [
+    ...documentationTools,
     ...connectionTools,
     ...resourceManagementTools,
     ...crudTools,
@@ -37,6 +39,7 @@ export function getAllTools() {
  */
 export function createAllHandlers(server) {
   return {
+    ...createDocumentationHandlers(server),
     ...createConnectionHandlers(server),
     ...createResourceManagementHandlers(server),
     ...createCrudHandlers(server),
@@ -54,6 +57,7 @@ export function createAllHandlers(server) {
  */
 export function getToolsByCategory() {
   return {
+    documentation: documentationTools,
     connection: connectionTools,
     resources: resourceManagementTools,
     crud: crudTools,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "11.3.2",
+  "version": "12.0.0",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",
@@ -72,6 +72,7 @@
     "dotenv": "^17.2.3",
     "fastest-validator": "^1.19.1",
     "flat": "^6.0.1",
+    "glob": "^11.0.3",
     "json-stable-stringify": "^1.3.0",
     "lodash-es": "^4.17.21",
     "nanoid": "5.1.6"
@@ -79,9 +80,12 @@
   "peerDependencies": {
     "@aws-sdk/client-sqs": "^3.0.0",
     "@google-cloud/bigquery": "^7.0.0",
+    "@hono/node-server": "^1.0.0",
+    "@hono/swagger-ui": "^0.5.0",
     "amqplib": "^0.10.8",
-    "pg": "^8.0.0",
-    "uuid": "^9.0.0"
+    "hono": "^4.0.0",
+    "node-cron": "^4.0.0",
+    "pg": "^8.0.0"
   },
   "peerDependenciesMeta": {
     "@aws-sdk/client-sqs": {
@@ -90,19 +94,30 @@
     "@google-cloud/bigquery": {
       "optional": true
     },
-    "pg": {
+    "@hono/node-server": {
       "optional": true
     },
-    "uuid": {
+    "@hono/swagger-ui": {
+      "optional": true
+    },
+    "pg": {
       "optional": true
     },
     "amqplib": {
       "optional": true
+    },
+    "hono": {
+      "optional": true
+    },
+    "node-cron": {
+      "optional": true
     }
   },
   "devDependencies": {
+    "@aws-sdk/client-sqs": "^3.0.0",
     "@babel/core": "^7.28.4",
     "@babel/preset-env": "^7.28.3",
+    "@google-cloud/bigquery": "^7.0.0",
     "@rollup/plugin-commonjs": "^28.0.6",
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^16.0.2",
@@ -110,6 +125,7 @@
     "@rollup/plugin-terser": "^0.4.4",
     "@types/node": "24.7.0",
     "@xenova/transformers": "^2.17.2",
+    "amqplib": "^0.10.8",
     "babel-loader": "^10.0.0",
     "chalk": "^5.6.2",
     "cli-table3": "^0.6.5",
@@ -117,8 +133,10 @@
     "esbuild": "^0.25.10",
     "inquirer": "^12.9.6",
     "jest": "^30.2.0",
+    "node-cron": "^4.0.0",
     "node-loader": "^2.1.0",
     "ora": "^9.0.0",
+    "pg": "^8.0.0",
     "pkg": "^5.8.1",
     "rollup": "^4.52.4",
     "rollup-plugin-copy": "^3.5.0",
@@ -128,6 +146,7 @@
     "rollup-plugin-terser": "^7.0.2",
     "tsx": "^4.20.6",
     "typescript": "5.9.3",
+    "uuid": "^13.0.0",
     "webpack": "^5.102.1",
     "webpack-cli": "^6.0.1"
   },
@@ -155,6 +174,8 @@
     "release:check": "./scripts/pre-release-check.sh",
     "validate:types": "pnpm run test:ts && echo 'TypeScript definitions are valid!'",
     "test:ts:runtime": "tsx tests/typescript/types-runtime-simple.ts",
-    "test:mcp": "node mcp/entrypoint.js --help"
+    "test:mcp": "node mcp/entrypoint.js --help",
+    "install:peers": "pnpm add -D @aws-sdk/client-sqs @google-cloud/bigquery amqplib node-cron pg",
+    "install:peers:script": "./scripts/install-peer-deps.sh"
   }
 }

package/src/behaviors/user-managed.js

@@ -1,6 +1,7 @@
 import { calculateTotalSize } from '../concerns/calculator.js';
 import { calculateEffectiveLimit } from '../concerns/calculator.js';
 import { S3_METADATA_LIMIT_BYTES } from './enforce-limits.js';
+import tryFn from '../concerns/try-fn.js';
 
 /**
  * User Managed Behavior Configuration Documentation
@@ -149,15 +150,21 @@ export async function handleUpsert({ resource, id, data, mappedData, originalDat
 export async function handleGet({ resource, metadata, body }) {
   // If body contains data, parse it and merge with metadata
   if (body && body.trim() !== '') {
-    try {
+    const [ok, error, result] = tryFn(() => {
       const bodyData = JSON.parse(body);
       // Merge body data with metadata, with metadata taking precedence
-      const mergedData = {
-        ...bodyData,
-        ...metadata
+      return {
+        metadata: {
+          ...bodyData,
+          ...metadata
+        },
+        body
       };
-      return { metadata: mergedData, body };
-    } catch (error) {
+    });
+
+    if (ok) {
+      return result;
+    } else {
       // If parsing fails, return original metadata and body
       return { metadata, body };
     }