s3db.js 13.3.0 → 13.4.0

package/src/plugins/fulltext.plugin.js

@@ -1,3 +1,487 @@
+/**
+ * # FullTextPlugin - Full-Text Search for s3db.js
+ *
+ * ## Overview
+ *
+ * The FullTextPlugin adds powerful full-text search capabilities to s3db.js, automatically
+ * indexing specified fields and providing fast, flexible search across your resources.
+ *
+ * ## Features
+ *
+ * 1. **Automatic Indexing** - Automatically indexes text fields on insert/update/delete
+ * 2. **Configurable Fields** - Choose which fields to index per resource
+ * 3. **Tokenization** - Intelligent word tokenization with configurable minimum length
+ * 4. **Partial Matching** - Support for both exact and partial word matching
+ * 5. **Relevance Scoring** - Results ranked by relevance score
+ * 6. **Persistent Indexes** - Indexes stored in S3 and loaded on startup
+ * 7. **Incremental Updates** - Only changed indexes are saved (dirty tracking)
+ * 8. **Index Management** - Rebuild, clear, and get statistics for indexes
+ *
+ * ## Configuration
+ *
+ * ```javascript
+ * import { Database } from 's3db.js';
+ * import { FullTextPlugin } from 's3db.js/plugins/fulltext';
+ *
+ * // Basic configuration
+ * const db = new Database({
+ *   connectionString: 's3://bucket/db'
+ * });
+ *
+ * await db.use(new FullTextPlugin({
+ *   minWordLength: 3,      // Minimum word length to index (default: 3)
+ *   maxResults: 100,       // Maximum search results (default: 100)
+ *   fields: ['title', 'description', 'content']  // Fields to index
+ * }));
+ *
+ * // Per-resource field mapping
+ * await db.use(new FullTextPlugin({
+ *   minWordLength: 2,      // Index shorter words
+ *   fields: {
+ *     users: ['name', 'email', 'bio'],
+ *     products: ['name', 'description', 'category'],
+ *     articles: ['title', 'content', 'tags']
+ *   }
+ * }));
+ * ```
+ *
+ * ## Usage Examples
+ *
+ * ### Basic Search
+ *
+ * ```javascript
+ * const db = new Database({ connectionString: 's3://bucket/db' });
+ * await db.use(new FullTextPlugin({
+ *   fields: ['title', 'content']
+ * }));
+ * await db.start();
+ *
+ * const articles = await db.createResource({
+ *   name: 'articles',
+ *   attributes: {
+ *     title: 'string',
+ *     content: 'string',
+ *     author: 'string'
+ *   }
+ * });
+ *
+ * // Insert articles (automatically indexed)
+ * await articles.insert({
+ *   id: 'a1',
+ *   title: 'Getting Started with S3DB',
+ *   content: 'S3DB is a document database built on AWS S3...',
+ *   author: 'John Doe'
+ * });
+ *
+ * // Search articles
+ * const fulltextPlugin = db.plugins.FullTextPlugin;
+ * const results = await fulltextPlugin.searchRecords('articles', 'S3DB database');
+ *
+ * console.log(results);
+ * // [
+ * //   {
+ * //     id: 'a1',
+ * //     title: 'Getting Started with S3DB',
+ * //     content: 'S3DB is a document database...',
+ * //     _searchScore: 2
+ * //   }
+ * // ]
+ * ```
+ *
+ * ### Search with Options
+ *
+ * ```javascript
+ * const fulltextPlugin = db.plugins.FullTextPlugin;
+ *
+ * // Exact match search
+ * const exact = await fulltextPlugin.searchRecords('articles', 'database', {
+ *   exactMatch: true,
+ *   limit: 10
+ * });
+ *
+ * // Partial match search (default)
+ * const partial = await fulltextPlugin.searchRecords('articles', 'data', {
+ *   exactMatch: false,
+ *   limit: 20
+ * });
+ *
+ * // Search specific fields
+ * const titleOnly = await fulltextPlugin.searchRecords('articles', 'S3DB', {
+ *   fields: ['title'],  // Search only title field
+ *   limit: 5
+ * });
+ *
+ * // Paginated search
+ * const page2 = await fulltextPlugin.searchRecords('articles', 'database', {
+ *   limit: 10,
+ *   offset: 10  // Skip first 10 results
+ * });
+ * ```
+ *
+ * ### Search IDs Only
+ *
+ * ```javascript
+ * // Get only record IDs and scores (faster)
+ * const idResults = await fulltextPlugin.search('articles', 'database');
+ *
+ * console.log(idResults);
+ * // [
+ * //   { recordId: 'a1', score: 3 },
+ * //   { recordId: 'a2', score: 2 },
+ * //   { recordId: 'a3', score: 1 }
+ * // ]
+ *
+ * // Fetch records manually if needed
+ * const records = await articles.getMany(idResults.map(r => r.recordId));
+ * ```
+ *
+ * ### Index Management
+ *
+ * ```javascript
+ * const fulltextPlugin = db.plugins.FullTextPlugin;
+ *
+ * // Rebuild index for a resource
+ * await fulltextPlugin.rebuildIndex('articles');
+ *
+ * // Rebuild all indexes
+ * await fulltextPlugin.rebuildAllIndexes();
+ *
+ * // Rebuild with timeout
+ * await fulltextPlugin.rebuildAllIndexes({ timeout: 30000 }); // 30 seconds
+ *
+ * // Get index statistics
+ * const stats = await fulltextPlugin.getIndexStats();
+ * console.log(stats);
+ * // {
+ * //   totalIndexes: 1523,
+ * //   totalWords: 245,
+ * //   resources: {
+ * //     articles: {
+ * //       totalRecords: 50,
+ * //       totalWords: 150,
+ * //       fields: {
+ * //         title: { words: 75, totalOccurrences: 100 },
+ * //         content: { words: 75, totalOccurrences: 200 }
+ * //       }
+ * //     }
+ * //   }
+ * // }
+ *
+ * // Clear specific resource index
+ * await fulltextPlugin.clearIndex('articles');
+ *
+ * // Clear all indexes
+ * await fulltextPlugin.clearAllIndexes();
+ * ```
+ *
+ * ## Best Practices
+ *
+ * ### 1. Choose Fields Wisely
+ *
+ * ```javascript
+ * // DON'T: Index all fields (wastes storage)
+ * await db.use(new FullTextPlugin({
+ *   fields: ['id', 'createdAt', 'updatedAt', 'title', 'content']  // ❌
+ * }));
+ *
+ * // DO: Index only searchable text fields
+ * await db.use(new FullTextPlugin({
+ *   fields: ['title', 'content', 'tags']  // ✅
+ * }));
+ * ```
+ *
+ * ### 2. Configure Minimum Word Length
+ *
+ * ```javascript
+ * // For general text (articles, blogs)
+ * await db.use(new FullTextPlugin({
+ *   minWordLength: 3  // Skip "a", "an", "the", etc.
+ * }));
+ *
+ * // For technical content (code, IDs)
+ * await db.use(new FullTextPlugin({
+ *   minWordLength: 2  // Allow shorter terms like "id", "db"
+ * }));
+ *
+ * // For specialized content (medical, legal)
+ * await db.use(new FullTextPlugin({
+ *   minWordLength: 4  // More selective indexing
+ * }));
+ * ```
+ *
+ * ### 3. Rebuild Indexes After Schema Changes
+ *
+ * ```javascript
+ * // After changing indexed fields
+ * await db.use(new FullTextPlugin({
+ *   fields: ['title', 'content', 'summary']  // Added 'summary'
+ * }));
+ *
+ * // Rebuild indexes to include new field
+ * const fulltextPlugin = db.plugins.FullTextPlugin;
+ * await fulltextPlugin.rebuildAllIndexes();
+ * ```
+ *
+ * ### 4. Use Exact Match for Precision
+ *
+ * ```javascript
+ * // For user search: partial match (more results)
+ * const userSearch = await fulltextPlugin.searchRecords('articles', query, {
+ *   exactMatch: false
+ * });
+ *
+ * // For filtering: exact match (precise results)
+ * const filtered = await fulltextPlugin.searchRecords('articles', 'database', {
+ *   exactMatch: true
+ * });
+ * ```
+ *
+ * ## Performance Considerations
+ *
+ * ### Indexing Performance
+ *
+ * - **Insert**: +10-50ms per record (depending on text length)
+ * - **Update**: +20-100ms per record (remove old + add new index)
+ * - **Delete**: +10-30ms per record (remove from index)
+ * - **Storage**: ~100-500 bytes per indexed word
+ *
+ * ### Search Performance
+ *
+ * | Records | Indexed Words | Search Time |
+ * |---------|---------------|-------------|
+ * | 1,000 | 5,000 | ~10ms |
+ * | 10,000 | 50,000 | ~50ms |
+ * | 100,000 | 500,000 | ~200ms |
+ *
+ * ### Optimization Tips
+ *
+ * ```javascript
+ * // 1. Use search() instead of searchRecords() when you don't need full records
+ * const ids = await fulltextPlugin.search('articles', 'database');  // Fast
+ * const records = await fulltextPlugin.searchRecords('articles', 'database');  // Slower
+ *
+ * // 2. Limit results
+ * const results = await fulltextPlugin.searchRecords('articles', 'database', {
+ *   limit: 20  // Faster than fetching 100+ results
+ * });
+ *
+ * // 3. Search specific fields
+ * const titleResults = await fulltextPlugin.searchRecords('articles', 'database', {
+ *   fields: ['title']  // Faster than searching all fields
+ * });
+ *
+ * // 4. Use pagination for large result sets
+ * for (let offset = 0; offset < total; offset += 50) {
+ *   const page = await fulltextPlugin.searchRecords('articles', 'database', {
+ *     limit: 50,
+ *     offset
+ *   });
+ *   processPage(page);
+ * }
+ * ```
+ *
+ * ## Troubleshooting
+ *
+ * ### Search Returns No Results
+ *
+ * ```javascript
+ * // Check if fields are configured
+ * const plugin = db.plugins.FullTextPlugin;
+ * console.log(plugin.config.fields);  // Should include the fields you're searching
+ *
+ * // Check index statistics
+ * const stats = await plugin.getIndexStats();
+ * console.log(stats.resources.articles);  // Should show indexed words
+ *
+ * // Rebuild index if needed
+ * await plugin.rebuildIndex('articles');
+ * ```
+ *
+ * ### Search Too Slow
+ *
+ * ```javascript
+ * // Solution 1: Reduce minWordLength to index fewer words
+ * await db.use(new FullTextPlugin({
+ *   minWordLength: 4  // More selective
+ * }));
+ *
+ * // Solution 2: Limit search fields
+ * const results = await plugin.searchRecords('articles', query, {
+ *   fields: ['title']  // Search only title, not content
+ * });
+ *
+ * // Solution 3: Use exact match
+ * const results = await plugin.searchRecords('articles', query, {
+ *   exactMatch: true  // Faster than partial matching
+ * });
+ * ```
+ *
+ * ### Index Growing Too Large
+ *
+ * ```javascript
+ * // Check index size
+ * const stats = await plugin.getIndexStats();
+ * console.log(`Total indexes: ${stats.totalIndexes}`);
+ * console.log(`Total words: ${stats.totalWords}`);
+ *
+ * // Solution 1: Increase minWordLength
+ * await db.use(new FullTextPlugin({
+ *   minWordLength: 4  // Index fewer words
+ * }));
+ * await plugin.rebuildAllIndexes();
+ *
+ * // Solution 2: Index fewer fields
+ * await db.use(new FullTextPlugin({
+ *   fields: ['title']  // Don't index long content fields
+ * }));
+ * await plugin.rebuildAllIndexes();
+ *
+ * // Solution 3: Clear old indexes
+ * await plugin.clearIndex('old_resource');
+ * ```
+ *
+ * ### Indexes Not Persisting
+ *
+ * ```javascript
+ * // Indexes save automatically on plugin stop
+ * await db.stop();  // Ensures indexes are saved
+ *
+ * // Or manually save
+ * await plugin.saveIndexes();
+ *
+ * // Check if index resource exists
+ * console.log(db.resources.plg_fulltext_indexes);  // Should exist
+ * ```
+ *
+ * ## Real-World Use Cases
+ *
+ * ### 1. Article/Blog Search
+ *
+ * ```javascript
+ * const plugin = new FullTextPlugin({
+ *   fields: ['title', 'content', 'tags'],
+ *   minWordLength: 3
+ * });
+ *
+ * // User searches for "javascript database"
+ * const results = await plugin.searchRecords('articles', 'javascript database', {
+ *   limit: 10
+ * });
+ *
+ * // Display results with highlights
+ * results.forEach(article => {
+ *   console.log(`${article.title} (score: ${article._searchScore})`);
+ * });
+ * ```
+ *
+ * ### 2. Product Search
+ *
+ * ```javascript
+ * const plugin = new FullTextPlugin({
+ *   fields: ['name', 'description', 'category', 'brand'],
+ *   minWordLength: 2
+ * });
+ *
+ * // Search for "laptop gaming"
+ * const products = await plugin.searchRecords('products', 'laptop gaming', {
+ *   limit: 20
+ * });
+ *
+ * // Filter by category after search
+ * const electronics = products.filter(p => p.category === 'Electronics');
+ * ```
+ *
+ * ### 3. User Directory Search
+ *
+ * ```javascript
+ * const plugin = new FullTextPlugin({
+ *   fields: ['name', 'email', 'department', 'title'],
+ *   minWordLength: 2
+ * });
+ *
+ * // Search for "john engineer"
+ * const users = await plugin.searchRecords('users', 'john engineer', {
+ *   limit: 10
+ * });
+ * ```
+ *
+ * ### 4. Documentation Search
+ *
+ * ```javascript
+ * const plugin = new FullTextPlugin({
+ *   fields: ['title', 'content', 'category'],
+ *   minWordLength: 3
+ * });
+ *
+ * // Search docs with exact match for technical terms
+ * const exactResults = await plugin.searchRecords('docs', 'insert()', {
+ *   exactMatch: true
+ * });
+ *
+ * // Fallback to partial match if no results
+ * if (exactResults.length === 0) {
+ *   const partialResults = await plugin.searchRecords('docs', 'insert', {
+ *     exactMatch: false
+ *   });
+ * }
+ * ```
+ *
+ * ## API Reference
+ *
+ * ### Constructor Options
+ *
+ * - `minWordLength` (number, default: 3) - Minimum word length to index
+ * - `maxResults` (number, default: 100) - Maximum search results
+ * - `fields` (string[] | object) - Fields to index (array or per-resource mapping)
+ *
+ * ### Methods
+ *
+ * - `search(resourceName, query, options)` - Search and return IDs with scores
+ * - `searchRecords(resourceName, query, options)` - Search and return full records
+ * - `rebuildIndex(resourceName)` - Rebuild index for a resource
+ * - `rebuildAllIndexes(options)` - Rebuild all indexes
+ * - `getIndexStats()` - Get index statistics
+ * - `clearIndex(resourceName)` - Clear specific resource index
+ * - `clearAllIndexes()` - Clear all indexes
+ * - `saveIndexes()` - Manually save indexes to S3
+ *
+ * ### Search Options
+ *
+ * ```typescript
+ * interface SearchOptions {
+ *   fields?: string[];      // Specific fields to search
+ *   limit?: number;         // Max results (default: maxResults from config)
+ *   offset?: number;        // Pagination offset (default: 0)
+ *   exactMatch?: boolean;   // Exact vs partial matching (default: false)
+ * }
+ * ```
+ *
+ * ### Search Result Structure
+ *
+ * ```typescript
+ * // search() returns
+ * interface SearchResult {
+ *   recordId: string;
+ *   score: number;  // Higher = more relevant
+ * }
+ *
+ * // searchRecords() returns
+ * interface SearchRecord extends ResourceRecord {
+ *   _searchScore: number;  // Added to each record
+ * }
+ * ```
+ *
+ * ## Notes
+ *
+ * - Indexes are stored in `plg_fulltext_indexes` resource
+ * - Tokenization preserves accented characters (é, ñ, etc.)
+ * - Case-insensitive search
+ * - Special characters are removed during tokenization
+ * - Nested field paths supported (e.g., 'profile.bio')
+ * - Indexes save automatically on plugin stop
+ * - Dirty tracking ensures only changed indexes are saved
+ */
+
 import { Plugin } from "./plugin.class.js";
 import tryFn from "../concerns/try-fn.js";
 import { FulltextError } from "./fulltext.errors.js";