s3db.js 7.4.2 → 8.0.0

package/src/plugins/cache.plugin.js

@@ -28,12 +28,12 @@ export class CachePlugin extends Plugin {
 
   async onSetup() {
     // Initialize cache driver
-    if (this.config.driver) {
-      // Use custom driver if provided
+    if (this.config.driver && typeof this.config.driver === 'object') {
+      // Use custom driver instance if provided
       this.driver = this.config.driver;
-    } else if (this.config.driverType === 'memory') {
+    } else if (this.config.driver === 'memory') {
       this.driver = new MemoryCache(this.config.memoryOptions || {});
-    } else if (this.config.driverType === 'filesystem') {
+    } else if (this.config.driver === 'filesystem') {
       // Use partition-aware filesystem cache if enabled
       if (this.config.partitionAware) {
         this.driver = new PartitionAwareFilesystemCache({
@@ -50,13 +50,23 @@ export class CachePlugin extends Plugin {
       this.driver = new S3Cache({ client: this.database.client, ...(this.config.s3Options || {}) });
     }
 
-    // Install database proxy for new resources
-    this.installDatabaseProxy();
+    // Use database hooks instead of method overwriting
+    this.installDatabaseHooks();
     
     // Install hooks for existing resources
     this.installResourceHooks();
   }
 
+  /**
+   * Install database hooks to handle resource creation/updates
+   */
+  installDatabaseHooks() {
+    // Hook into resource creation to install cache middleware
+    this.database.addHook('afterCreateResource', async ({ resource }) => {
+      this.installResourceHooksForResource(resource);
+    });
+  }
+
   async onStart() {
     // Plugin is ready
   }
@@ -65,27 +75,7 @@ export class CachePlugin extends Plugin {
     // Cleanup if needed
   }
 
-  installDatabaseProxy() {
-    if (this.database._cacheProxyInstalled) {
-      return; // Already installed
-    }
-    
-    const installResourceHooks = this.installResourceHooks.bind(this);
-    
-    // Store original method
-    this.database._originalCreateResourceForCache = this.database.createResource;
-    
-    // Create new method that doesn't call itself
-    this.database.createResource = async function (...args) {
-      const resource = await this._originalCreateResourceForCache(...args);
-      installResourceHooks(resource);
-      return resource;
-    };
-    
-    // Mark as installed
-    this.database._cacheProxyInstalled = true;
-  }
-
+  // Remove the old installDatabaseProxy method
   installResourceHooks() {
     for (const resource of Object.values(this.database.resources)) {
       this.installResourceHooksForResource(resource);
@@ -126,10 +116,12 @@ export class CachePlugin extends Plugin {
       };
     }
 
-    // List of methods to cache
+    // Expanded list of methods to cache (including previously missing ones)
     const cacheMethods = [
-      'count', 'listIds', 'getMany', 'getAll', 'page', 'list', 'get'
+      'count', 'listIds', 'getMany', 'getAll', 'page', 'list', 'get',
+      'exists', 'content', 'hasContent', 'query', 'getFromPartition'
     ];
+    
     for (const method of cacheMethods) {
       resource.useMiddleware(method, async (ctx, next) => {
         // Build cache key
@@ -142,11 +134,29 @@ export class CachePlugin extends Plugin {
         } else if (method === 'list' || method === 'listIds' || method === 'count') {
           const { partition, partitionValues } = ctx.args[0] || {};
           key = await resource.cacheKeyFor({ action: method, partition, partitionValues });
+        } else if (method === 'query') {
+          const filter = ctx.args[0] || {};
+          const options = ctx.args[1] || {};
+          key = await resource.cacheKeyFor({ 
+            action: method, 
+            params: { filter, options: { limit: options.limit, offset: options.offset } },
+            partition: options.partition,
+            partitionValues: options.partitionValues
+          });
+        } else if (method === 'getFromPartition') {
+          const { id, partitionName, partitionValues } = ctx.args[0] || {};
+          key = await resource.cacheKeyFor({ 
+            action: method, 
+            params: { id, partitionName }, 
+            partition: partitionName, 
+            partitionValues 
+          });
         } else if (method === 'getAll') {
           key = await resource.cacheKeyFor({ action: method });
-        } else if (method === 'get') {
+        } else if (['get', 'exists', 'content', 'hasContent'].includes(method)) {
           key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
         }
+        
         // Try cache with partition awareness
         let cached;
         if (this.driver instanceof PartitionAwareFilesystemCache) {
@@ -156,6 +166,14 @@ export class CachePlugin extends Plugin {
             const args = ctx.args[0] || {};
             partition = args.partition;
             partitionValues = args.partitionValues;
+          } else if (method === 'query') {
+            const options = ctx.args[1] || {};
+            partition = options.partition;
+            partitionValues = options.partitionValues;
+          } else if (method === 'getFromPartition') {
+            const { partitionName, partitionValues: pValues } = ctx.args[0] || {};
+            partition = partitionName;
+            partitionValues = pValues;
           }
           
           const [ok, err, result] = await tryFn(() => resource.cache._get(key, {
@@ -194,8 +212,8 @@ export class CachePlugin extends Plugin {
       });
     }
 
-    // List of methods to clear cache on write
-    const writeMethods = ['insert', 'update', 'delete', 'deleteMany'];
+    // List of methods to clear cache on write (expanded to include new methods)
+    const writeMethods = ['insert', 'update', 'delete', 'deleteMany', 'setContent', 'deleteContent', 'replace'];
     for (const method of writeMethods) {
       resource.useMiddleware(method, async (ctx, next) => {
         const result = await next();
@@ -211,6 +229,12 @@ export class CachePlugin extends Plugin {
             if (ok && full) data = full;
           }
           await this.clearCacheForResource(resource, data);
+        } else if (method === 'setContent' || method === 'deleteContent') {
+          const id = ctx.args[0]?.id || ctx.args[0];
+          await this.clearCacheForResource(resource, { id });
+        } else if (method === 'replace') {
+          const id = ctx.args[0];
+          await this.clearCacheForResource(resource, { id, ...ctx.args[1] });
         } else if (method === 'deleteMany') {
           // After all deletions, clear all aggregate and partition caches
           await this.clearCacheForResource(resource);
@@ -225,28 +249,52 @@ export class CachePlugin extends Plugin {
     
     const keyPrefix = `resource=${resource.name}`;
     
-    // Always clear main cache for this resource
-    await resource.cache.clear(keyPrefix);
-    
-    // Only clear partition cache if partitions are enabled AND resource has partitions AND includePartitions is true
-    if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
-      if (!data) {
-        // If no data, clear all partition caches
-        for (const partitionName of Object.keys(resource.config.partitions)) {
-          const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
-          await resource.cache.clear(partitionKeyPrefix);
+    // For specific operations, only clear relevant cache entries
+    if (data && data.id) {
+      // Clear specific item caches for this ID
+      const itemSpecificMethods = ['get', 'exists', 'content', 'hasContent'];
+      for (const method of itemSpecificMethods) {
+        try {
+          const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
+          await resource.cache.clear(specificKey.replace('.json.gz', ''));
+        } catch (error) {
+          // Ignore cache clearing errors for individual items
         }
-      } else {
+      }
+      
+      // Clear partition-specific caches if this resource has partitions
+      if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
         const partitionValues = this.getPartitionValues(data, resource);
         for (const [partitionName, values] of Object.entries(partitionValues)) {
-          // Only clear partition cache if there are actual values
           if (values && Object.keys(values).length > 0 && Object.values(values).some(v => v !== null && v !== undefined)) {
-            const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
-            await resource.cache.clear(partitionKeyPrefix);
+            try {
+              const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
+              await resource.cache.clear(partitionKeyPrefix);
+            } catch (error) {
+              // Ignore partition cache clearing errors
+            }
           }
         }
       }
     }
+    
+    // Clear aggregate caches more broadly to ensure all variants are cleared
+    try {
+      // Clear all cache entries for this resource - this ensures aggregate methods are invalidated
+      await resource.cache.clear(keyPrefix);
+    } catch (error) {
+      // If broad clearing fails, try specific method clearing
+      const aggregateMethods = ['count', 'list', 'listIds', 'getAll', 'page', 'query'];
+      for (const method of aggregateMethods) {
+        try {
+          // Try multiple key patterns to ensure we catch all variations
+          await resource.cache.clear(`${keyPrefix}/action=${method}`);
+          await resource.cache.clear(`resource=${resource.name}/action=${method}`);
+        } catch (methodError) {
+          // Ignore individual method clearing errors
+        }
+      }
+    }
   }
 
   async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
@@ -277,7 +325,7 @@ export class CachePlugin extends Plugin {
   async hashParams(params) {
     const sortedParams = Object.keys(params)
       .sort()
-      .map(key => `${key}:${params[key]}`)
+      .map(key => `${key}:${JSON.stringify(params[key])}`) // Use JSON.stringify for complex objects
       .join('|') || 'empty';
     
     return await sha256(sortedParams);

package/src/plugins/fulltext.plugin.js

@@ -34,6 +34,10 @@ export class FullTextPlugin extends Plugin {
     // Load existing indexes
     await this.loadIndexes();
     
+    // Use database hooks for automatic resource discovery
+    this.installDatabaseHooks();
+    
+    // Install hooks for existing resources
     this.installIndexingHooks();
   }
 
@@ -44,6 +48,9 @@ export class FullTextPlugin extends Plugin {
   async stop() {
     // Save indexes before stopping
     await this.saveIndexes();
+    
+    // Remove database hooks
+    this.removeDatabaseHooks();
   }
 
   async loadIndexes() {
@@ -86,6 +93,20 @@ export class FullTextPlugin extends Plugin {
     });
   }
 
+  installDatabaseHooks() {
+    // Use the new database hooks system for automatic resource discovery
+    this.database.addHook('afterCreateResource', (resource) => {
+      if (resource.name !== 'fulltext_indexes') {
+        this.installResourceHooks(resource);
+      }
+    });
+  }
+
+  removeDatabaseHooks() {
+    // Remove the hook we added
+    this.database.removeHook('afterCreateResource', this.installResourceHooks.bind(this));
+  }
+
   installIndexingHooks() {
     // Register plugin with database
     if (!this.database.plugins) {

package/src/plugins/metrics.plugin.js

@@ -86,7 +86,10 @@ export class MetricsPlugin extends Plugin {
       this.performanceResource = database.resources.performance_logs;
     }
 
-    // Install hooks for all resources except metrics resources
+    // Use database hooks for automatic resource discovery
+    this.installDatabaseHooks();
+    
+    // Install hooks for existing resources
     this.installMetricsHooks();
     
     // Disable flush timer during tests to avoid side effects
@@ -100,16 +103,28 @@ export class MetricsPlugin extends Plugin {
   }
 
   async stop() {
-    // Stop flush timer and flush remaining metrics
+    // Stop flush timer
     if (this.flushTimer) {
       clearInterval(this.flushTimer);
       this.flushTimer = null;
     }
     
-    // Don't flush metrics during tests
-    if (typeof process !== 'undefined' && process.env.NODE_ENV !== 'test') {
-      await this.flushMetrics();
-    }
+    // Remove database hooks
+    this.removeDatabaseHooks();
+  }
+
+  installDatabaseHooks() {
+    // Use the new database hooks system for automatic resource discovery
+    this.database.addHook('afterCreateResource', (resource) => {
+      if (resource.name !== 'metrics' && resource.name !== 'error_logs' && resource.name !== 'performance_logs') {
+        this.installResourceHooks(resource);
+      }
+    });
+  }
+
+  removeDatabaseHooks() {
+    // Remove the hook we added
+    this.database.removeHook('afterCreateResource', this.installResourceHooks.bind(this));
   }
 
   installMetricsHooks() {

package/src/plugins/replicator.plugin.js

@@ -164,6 +164,13 @@ export class ReplicatorPlugin extends Plugin {
     return filtered;
   }
 
+  async getCompleteData(resource, data) {
+    // Always get the complete record from the resource to ensure we have all data
+    // This handles all behaviors: body-overflow, truncate-data, body-only, etc.
+    const [ok, err, completeRecord] = await tryFn(() => resource.get(data.id));
+    return ok ? completeRecord : data;
+  }
+
   installEventListeners(resource, database, plugin) {
     if (!resource || this.eventListenersInstalled.has(resource.name) || 
         resource.name === this.config.replicatorLogResource) {
@@ -186,8 +193,10 @@ export class ReplicatorPlugin extends Plugin {
 
     resource.on('update', async (data, beforeData) => {
       const [ok, error] = await tryFn(async () => {
-        const completeData = { ...data, updatedAt: new Date().toISOString() };
-        await plugin.processReplicatorEvent('update', resource.name, completeData.id, completeData, beforeData);
+        // For updates, we need to get the complete updated record, not just the changed fields
+        const completeData = await plugin.getCompleteData(resource, data);
+        const dataWithTimestamp = { ...completeData, updatedAt: new Date().toISOString() };
+        await plugin.processReplicatorEvent('update', resource.name, completeData.id, dataWithTimestamp, beforeData);
       });
       
       if (!ok) {
@@ -214,76 +223,73 @@ export class ReplicatorPlugin extends Plugin {
     this.eventListenersInstalled.add(resource.name);
   }
 
-  /**
-   * Get complete data by always fetching the full record from the resource
-   * This ensures we always have the complete data regardless of behavior or data size
-   */
-  async getCompleteData(resource, data) {
-    // Always get the complete record from the resource to ensure we have all data
-    // This handles all behaviors: body-overflow, truncate-data, body-only, etc.
-    const [ok, err, completeRecord] = await tryFn(() => resource.get(data.id));
-    return ok ? completeRecord : data;
-  }
-
   async setup(database) {
     this.database = database;
-
-    const [initOk, initError] = await tryFn(async () => {
-      await this.initializeReplicators(database);
-    });
     
-    if (!initOk) {
-      if (this.config.verbose) {
-        console.warn(`[ReplicatorPlugin] Replicator initialization failed: ${initError.message}`);
+    // Create replicator log resource if enabled
+    if (this.config.persistReplicatorLog) {
+      const [ok, err, logResource] = await tryFn(() => database.createResource({
+        name: this.config.replicatorLogResource || 'replicator_logs',
+        attributes: {
+          id: 'string|required',
+          resource: 'string|required',
+          action: 'string|required',
+          data: 'json',
+          timestamp: 'number|required',
+          createdAt: 'string|required'
+        },
+        behavior: 'truncate-data'
+      }));
+      
+      if (ok) {
+        this.replicatorLogResource = logResource;
+      } else {
+        this.replicatorLogResource = database.resources[this.config.replicatorLogResource || 'replicator_logs'];
       }
-      this.emit('error', { operation: 'setup', error: initError.message });
-      throw initError;
     }
 
-    const [logOk, logError] = await tryFn(async () => {
-      if (this.config.replicatorLogResource) {
-        const logRes = await database.createResource({
-          name: this.config.replicatorLogResource,
-          behavior: 'body-overflow',
-          attributes: {
-            operation: 'string',
-            resourceName: 'string', 
-            recordId: 'string',
-            data: 'string',
-            error: 'string|optional',
-            replicator: 'string',
-            timestamp: 'string',
-            status: 'string'
-          }
-        });
-      }
-    });
+    // Initialize replicators
+    await this.initializeReplicators(database);
     
-    if (!logOk) {
-      if (this.config.verbose) {
-        console.warn(`[ReplicatorPlugin] Failed to create log resource ${this.config.replicatorLogResource}: ${logError.message}`);
+    // Use database hooks for automatic resource discovery
+    this.installDatabaseHooks();
+    
+    // Install event listeners for existing resources
+    for (const resource of Object.values(database.resources)) {
+      if (resource.name !== (this.config.replicatorLogResource || 'replicator_logs')) {
+        this.installEventListeners(resource, database, this);
       }
-      this.emit('replicator_log_resource_creation_error', {
-        resourceName: this.config.replicatorLogResource,
-        error: logError.message
-      });
     }
+  }
 
-    await this.uploadMetadataFile(database);
+  async start() {
+    // Plugin is ready
+  }
 
-    const originalCreateResource = database.createResource.bind(database);
-    database.createResource = async (config) => {
-      const resource = await originalCreateResource(config);
-      if (resource) {
-        this.installEventListeners(resource, database, this);
+  async stop() {
+    // Stop all replicators
+    for (const replicator of this.replicators || []) {
+      if (replicator && typeof replicator.cleanup === 'function') {
+        await replicator.cleanup();
       }
-      return resource;
-    };
-
-    for (const resourceName in database.resources) {
-      const resource = database.resources[resourceName];
-      this.installEventListeners(resource, database, this);
     }
+    
+    // Remove database hooks
+    this.removeDatabaseHooks();
+  }
+
+  installDatabaseHooks() {
+    // Use the new database hooks system for automatic resource discovery
+    this.database.addHook('afterCreateResource', (resource) => {
+      if (resource.name !== (this.config.replicatorLogResource || 'replicator_logs')) {
+        this.installEventListeners(resource, this.database, this);
+      }
+    });
+  }
+
+  removeDatabaseHooks() {
+    // Remove the hook we added
+    this.database.removeHook('afterCreateResource', this.installEventListeners.bind(this));
   }
 
   createReplicator(driver, config, resources, client) {
@@ -309,17 +315,6 @@ export class ReplicatorPlugin extends Plugin {
     }
   }
 
-  async start() {
-    // Plugin is ready
-  }
-
-  async stop() {
-    // Stop queue processing
-    // this.isProcessing = false; // Removed as per edit hint
-    // Process remaining queue items
-    // await this.processQueue(); // Removed as per edit hint
-  }
-
   async uploadMetadataFile(database) {
     if (typeof database.uploadMetadataFile === 'function') {
       await database.uploadMetadataFile();

package/src/resource.class.js

@@ -2484,10 +2484,11 @@ export class Resource extends EventEmitter {
   _initMiddleware() {
     // Map of methodName -> array of middleware functions
     this._middlewares = new Map();
-    // Supported methods for middleware
+    // Supported methods for middleware (expanded to include newly cached methods)
     this._middlewareMethods = [
       'get', 'list', 'listIds', 'getAll', 'count', 'page',
-      'insert', 'update', 'delete', 'deleteMany', 'exists', 'getMany'
+      'insert', 'update', 'delete', 'deleteMany', 'exists', 'getMany',
+      'content', 'hasContent', 'query', 'getFromPartition', 'setContent', 'deleteContent', 'replace'
     ];
     for (const method of this._middlewareMethods) {
       this._middlewares.set(method, []);

package/src/s3db.d.ts

@@ -6,6 +6,15 @@ declare module 's3db.js' {
   // CORE TYPES
   // ============================================================================
 
+  /** HTTP Client configuration for keep-alive and connection pooling */
+  export interface HttpClientOptions {
+    keepAlive?: boolean;
+    keepAliveMsecs?: number;
+    maxSockets?: number;
+    maxFreeSockets?: number;
+    timeout?: number;
+  }
+
   /** Main Database configuration */
   export interface DatabaseConfig {
     connectionString?: string;
@@ -23,6 +32,7 @@ declare module 's3db.js' {
     cache?: CacheConfig | boolean;
     plugins?: (PluginInterface | PluginFunction)[];
     client?: Client;
+    httpClientOptions?: HttpClientOptions;
   }
 
   /** Resource configuration */