s3db.js 11.2.2 → 11.2.4

package/src/plugins/eventual-consistency/index.js

@@ -17,7 +17,7 @@ import {
   runConsolidation
 } from "./consolidation.js";
 import { runGarbageCollection } from "./garbage-collection.js";
-import { updateAnalytics, getAnalytics, getMonthByDay, getDayByHour, getLastNDays, getYearByMonth, getYearByWeek, getMonthByWeek, getMonthByHour, getTopRecords, getYearByDay, getWeekByDay, getWeekByHour, getLastNHours, getLastNWeeks, getLastNMonths } from "./analytics.js";
+import { updateAnalytics, getAnalytics, getMonthByDay, getDayByHour, getLastNDays, getYearByMonth, getYearByWeek, getMonthByWeek, getMonthByHour, getTopRecords, getYearByDay, getWeekByDay, getWeekByHour, getLastNHours, getLastNWeeks, getLastNMonths, getRawEvents } from "./analytics.js";
 import { onInstall, onStart, onStop, watchForResource, completeFieldSetup } from "./install.js";
 
 export class EventualConsistencyPlugin extends Plugin {
@@ -531,6 +531,208 @@ export class EventualConsistencyPlugin extends Plugin {
   async getLastNMonths(resourceName, field, months = 12, options = {}) {
     return await getLastNMonths(resourceName, field, months, options, this.fieldHandlers);
   }
+
+  /**
+   * Get raw transaction events for custom aggregation
+   *
+   * This method provides direct access to the underlying transaction events,
+   * allowing developers to perform custom aggregations beyond the pre-built analytics.
+   * Useful for complex queries, custom metrics, or when you need the raw event data.
+   *
+   * @param {string} resourceName - Resource name
+   * @param {string} field - Field name
+   * @param {Object} options - Query options
+   * @param {string} options.recordId - Filter by specific record ID
+   * @param {string} options.startDate - Start date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
+   * @param {string} options.endDate - End date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
+   * @param {string} options.cohortDate - Filter by cohort date (YYYY-MM-DD)
+   * @param {string} options.cohortHour - Filter by cohort hour (YYYY-MM-DDTHH)
+   * @param {string} options.cohortMonth - Filter by cohort month (YYYY-MM)
+   * @param {boolean} options.applied - Filter by applied status (true/false/undefined for both)
+   * @param {string} options.operation - Filter by operation type ('add', 'sub', 'set')
+   * @param {number} options.limit - Maximum number of events to return
+   * @returns {Promise<Array>} Raw transaction events
+   *
+   * @example
+   * // Get all events for a specific record
+   * const events = await plugin.getRawEvents('wallets', 'balance', {
+   *   recordId: 'wallet1'
+   * });
+   *
+   * @example
+   * // Get events for a specific time range
+   * const events = await plugin.getRawEvents('wallets', 'balance', {
+   *   startDate: '2025-10-01',
+   *   endDate: '2025-10-31'
+   * });
+   *
+   * @example
+   * // Get only pending (unapplied) transactions
+   * const pending = await plugin.getRawEvents('wallets', 'balance', {
+   *   applied: false
+   * });
+   */
+  async getRawEvents(resourceName, field, options = {}) {
+    return await getRawEvents(resourceName, field, options, this.fieldHandlers);
+  }
+
+  /**
+   * Get diagnostics information about the plugin state
+   *
+   * This method provides comprehensive diagnostic information about the EventualConsistencyPlugin,
+   * including configured resources, field handlers, timers, and overall health status.
+   * Useful for debugging initialization issues, configuration problems, or runtime errors.
+   *
+   * @param {Object} options - Diagnostic options
+   * @param {string} options.resourceName - Optional: limit diagnostics to specific resource
+   * @param {string} options.field - Optional: limit diagnostics to specific field
+   * @param {boolean} options.includeStats - Include transaction statistics (default: false)
+   * @returns {Promise<Object>} Diagnostic information
+   *
+   * @example
+   * // Get overall plugin diagnostics
+   * const diagnostics = await plugin.getDiagnostics();
+   * console.log(diagnostics);
+   *
+   * @example
+   * // Get diagnostics for specific resource/field with stats
+   * const diagnostics = await plugin.getDiagnostics({
+   *   resourceName: 'wallets',
+   *   field: 'balance',
+   *   includeStats: true
+   * });
+   */
+  async getDiagnostics(options = {}) {
+    const { resourceName, field, includeStats = false } = options;
+
+    const diagnostics = {
+      plugin: {
+        name: 'EventualConsistencyPlugin',
+        initialized: this.database !== null && this.database !== undefined,
+        verbose: this.config.verbose || false,
+        timezone: this.config.cohort?.timezone || 'UTC',
+        consolidation: {
+          mode: this.config.consolidation?.mode || 'timer',
+          interval: this.config.consolidation?.interval || 60000,
+          batchSize: this.config.consolidation?.batchSize || 100
+        },
+        garbageCollection: {
+          enabled: this.config.garbageCollection?.enabled !== false,
+          retentionDays: this.config.garbageCollection?.retentionDays || 30,
+          interval: this.config.garbageCollection?.interval || 3600000
+        }
+      },
+      resources: [],
+      errors: [],
+      warnings: []
+    };
+
+    // Iterate through configured resources
+    for (const [resName, resourceHandlers] of this.fieldHandlers.entries()) {
+      // Skip if filtering by resource and this isn't it
+      if (resourceName && resName !== resourceName) {
+        continue;
+      }
+
+      const resourceDiag = {
+        name: resName,
+        fields: []
+      };
+
+      for (const [fieldName, handler] of resourceHandlers.entries()) {
+        // Skip if filtering by field and this isn't it
+        if (field && fieldName !== field) {
+          continue;
+        }
+
+        const fieldDiag = {
+          name: fieldName,
+          type: handler.type || 'counter',
+          analyticsEnabled: handler.analyticsResource !== null && handler.analyticsResource !== undefined,
+          resources: {
+            transaction: handler.transactionResource?.name || null,
+            target: handler.targetResource?.name || null,
+            analytics: handler.analyticsResource?.name || null
+          },
+          timers: {
+            consolidation: handler.consolidationTimer !== null && handler.consolidationTimer !== undefined,
+            garbageCollection: handler.garbageCollectionTimer !== null && handler.garbageCollectionTimer !== undefined
+          }
+        };
+
+        // Check for common issues
+        if (!handler.transactionResource) {
+          diagnostics.errors.push({
+            resource: resName,
+            field: fieldName,
+            issue: 'Missing transaction resource',
+            suggestion: 'Ensure plugin is installed and resources are created after plugin installation'
+          });
+        }
+
+        if (!handler.targetResource) {
+          diagnostics.warnings.push({
+            resource: resName,
+            field: fieldName,
+            issue: 'Missing target resource',
+            suggestion: 'Target resource may not have been created yet'
+          });
+        }
+
+        if (handler.analyticsResource && !handler.analyticsResource.name) {
+          diagnostics.errors.push({
+            resource: resName,
+            field: fieldName,
+            issue: 'Invalid analytics resource',
+            suggestion: 'Analytics resource exists but has no name - possible initialization failure'
+          });
+        }
+
+        // Include statistics if requested
+        if (includeStats && handler.transactionResource) {
+          try {
+            const [okPending, errPending, pendingTxns] = await handler.transactionResource.query({ applied: false }).catch(() => [false, null, []]);
+            const [okApplied, errApplied, appliedTxns] = await handler.transactionResource.query({ applied: true }).catch(() => [false, null, []]);
+
+            fieldDiag.stats = {
+              pendingTransactions: okPending ? (pendingTxns?.length || 0) : 'error',
+              appliedTransactions: okApplied ? (appliedTxns?.length || 0) : 'error',
+              totalTransactions: (okPending && okApplied) ? ((pendingTxns?.length || 0) + (appliedTxns?.length || 0)) : 'error'
+            };
+
+            if (handler.analyticsResource) {
+              const [okAnalytics, errAnalytics, analyticsRecords] = await handler.analyticsResource.list().catch(() => [false, null, []]);
+              fieldDiag.stats.analyticsRecords = okAnalytics ? (analyticsRecords?.length || 0) : 'error';
+            }
+          } catch (error) {
+            diagnostics.warnings.push({
+              resource: resName,
+              field: fieldName,
+              issue: 'Failed to fetch statistics',
+              error: error.message
+            });
+          }
+        }
+
+        resourceDiag.fields.push(fieldDiag);
+      }
+
+      if (resourceDiag.fields.length > 0) {
+        diagnostics.resources.push(resourceDiag);
+      }
+    }
+
+    // Overall health check
+    diagnostics.health = {
+      status: diagnostics.errors.length === 0 ? (diagnostics.warnings.length === 0 ? 'healthy' : 'warning') : 'error',
+      totalResources: diagnostics.resources.length,
+      totalFields: diagnostics.resources.reduce((sum, r) => sum + r.fields.length, 0),
+      errorCount: diagnostics.errors.length,
+      warningCount: diagnostics.warnings.length
+    };
+
+    return diagnostics;
+  }
 }
 
 export default EventualConsistencyPlugin;

package/src/plugins/fulltext.errors.js

@@ -0,0 +1,46 @@
+import { S3dbError } from '../errors.js';
+
+/**
+ * FulltextError - Errors related to fulltext search operations
+ *
+ * Used for fulltext search operations including:
+ * - Index creation and updates
+ * - Search query execution
+ * - Index configuration
+ * - Text analysis and tokenization
+ * - Search result ranking
+ *
+ * @extends S3dbError
+ */
+export class FulltextError extends S3dbError {
+  constructor(message, details = {}) {
+    const { resourceName, query, operation = 'unknown', ...rest } = details;
+
+    let description = details.description;
+    if (!description) {
+      description = `
+Fulltext Search Operation Error
+
+Operation: ${operation}
+${resourceName ? `Resource: ${resourceName}` : ''}
+${query ? `Query: ${query}` : ''}
+
+Common causes:
+1. Resource not indexed for fulltext search
+2. Invalid query syntax
+3. Index not built yet
+4. Search configuration missing
+5. Field not indexed
+
+Solution:
+Ensure resource is configured for fulltext search and index is built.
+
+Docs: https://github.com/forattini-dev/s3db.js/blob/main/docs/plugins/fulltext.md
+`.trim();
+    }
+
+    super(message, { ...rest, resourceName, query, operation, description });
+  }
+}
+
+export default FulltextError;

package/src/plugins/fulltext.plugin.js

@@ -1,5 +1,6 @@
 import Plugin from "./plugin.class.js";
 import tryFn from "../concerns/try-fn.js";
+import { FulltextError } from "./fulltext.errors.js";
 
 export class FullTextPlugin extends Plugin {
   constructor(options = {}) {
@@ -399,14 +400,20 @@ export class FullTextPlugin extends Plugin {
   // Search and return full records
   async searchRecords(resourceName, query, options = {}) {
     const searchResults = await this.search(resourceName, query, options);
-    
+
     if (searchResults.length === 0) {
       return [];
     }
 
     const resource = this.database.resources[resourceName];
     if (!resource) {
-      throw new Error(`Resource '${resourceName}' not found`);
+      throw new FulltextError(`Resource '${resourceName}' not found`, {
+        operation: 'searchRecords',
+        resourceName,
+        query,
+        availableResources: Object.keys(this.database.resources),
+        suggestion: 'Check resource name or ensure resource is created before searching'
+      });
     }
 
     const recordIds = searchResults.map(result => result.recordId);
@@ -430,7 +437,12 @@ export class FullTextPlugin extends Plugin {
   async rebuildIndex(resourceName) {
     const resource = this.database.resources[resourceName];
     if (!resource) {
-      throw new Error(`Resource '${resourceName}' not found`);
+      throw new FulltextError(`Resource '${resourceName}' not found`, {
+        operation: 'rebuildIndex',
+        resourceName,
+        availableResources: Object.keys(this.database.resources),
+        suggestion: 'Check resource name or ensure resource is created before rebuilding index'
+      });
     }
 
     // Clear existing indexes for this resource

package/src/plugins/metrics.errors.js

@@ -0,0 +1,46 @@
+import { S3dbError } from '../errors.js';
+
+/**
+ * MetricsError - Errors related to metrics operations
+ *
+ * Used for metrics operations including:
+ * - Metric collection and recording
+ * - Metric aggregation
+ * - Metric querying
+ * - Performance tracking
+ * - Statistics computation
+ *
+ * @extends S3dbError
+ */
+export class MetricsError extends S3dbError {
+  constructor(message, details = {}) {
+    const { metricName, operation = 'unknown', resourceName, ...rest } = details;
+
+    let description = details.description;
+    if (!description) {
+      description = `
+Metrics Operation Error
+
+Operation: ${operation}
+${metricName ? `Metric: ${metricName}` : ''}
+${resourceName ? `Resource: ${resourceName}` : ''}
+
+Common causes:
+1. Metric not configured
+2. Invalid metric value or type
+3. Metrics storage not accessible
+4. Aggregation function error
+5. Query parameters invalid
+
+Solution:
+Check metrics configuration and ensure proper initialization.
+
+Docs: https://github.com/forattini-dev/s3db.js/blob/main/docs/plugins/metrics.md
+`.trim();
+    }
+
+    super(message, { ...rest, metricName, operation, resourceName, description });
+  }
+}
+
+export default MetricsError;

package/src/plugins/queue-consumer.plugin.js

@@ -1,6 +1,7 @@
 import { Plugin } from './plugin.class.js';
 import { createConsumer } from './consumers/index.js';
 import tryFn from "../concerns/try-fn.js";
+import { QueueError } from "./queue.errors.js";
 
 // Example configuration for SQS:
 // const plugin = new QueueConsumerPlugin({
@@ -101,13 +102,32 @@ export class QueueConsumerPlugin extends Plugin {
 
     
     if (!resource) {
-      throw new Error('QueueConsumerPlugin: resource not found in message');
+      throw new QueueError('Resource not found in message', {
+        operation: 'handleMessage',
+        queueName: configuredResource,
+        messageBody: body,
+        suggestion: 'Ensure message includes a "resource" field specifying the target resource name'
+      });
     }
     if (!action) {
-      throw new Error('QueueConsumerPlugin: action not found in message');
+      throw new QueueError('Action not found in message', {
+        operation: 'handleMessage',
+        queueName: configuredResource,
+        resource,
+        messageBody: body,
+        suggestion: 'Ensure message includes an "action" field (insert, update, or delete)'
+      });
     }
     const resourceObj = this.database.resources[resource];
-    if (!resourceObj) throw new Error(`QueueConsumerPlugin: resource '${resource}' not found`);
+    if (!resourceObj) {
+      throw new QueueError(`Resource '${resource}' not found`, {
+        operation: 'handleMessage',
+        queueName: configuredResource,
+        resource,
+        availableResources: Object.keys(this.database.resources),
+        suggestion: 'Check resource name or ensure resource is created before consuming messages'
+      });
+    }
     
     let result;
     const [ok, err, res] = await tryFn(async () => {
@@ -119,7 +139,14 @@ export class QueueConsumerPlugin extends Plugin {
       } else if (action === 'delete') {
         result = await resourceObj.delete(data.id);
       } else {
-        throw new Error(`QueueConsumerPlugin: unsupported action '${action}'`);
+        throw new QueueError(`Unsupported action '${action}'`, {
+          operation: 'handleMessage',
+          queueName: configuredResource,
+          resource,
+          action,
+          supportedActions: ['insert', 'update', 'delete'],
+          suggestion: 'Use one of the supported actions: insert, update, or delete'
+        });
       }
       return result;
     });

package/src/plugins/queue.errors.js

@@ -0,0 +1,46 @@
+import { S3dbError } from '../errors.js';
+
+/**
+ * QueueError - Errors related to queue operations
+ *
+ * Used for queue operations including:
+ * - Message enqueueing and dequeueing
+ * - Queue consumer registration
+ * - Message processing
+ * - Dead letter queue handling
+ * - Queue configuration and management
+ *
+ * @extends S3dbError
+ */
+export class QueueError extends S3dbError {
+  constructor(message, details = {}) {
+    const { queueName, operation = 'unknown', messageId, ...rest } = details;
+
+    let description = details.description;
+    if (!description) {
+      description = `
+Queue Operation Error
+
+Operation: ${operation}
+${queueName ? `Queue: ${queueName}` : ''}
+${messageId ? `Message ID: ${messageId}` : ''}
+
+Common causes:
+1. Queue not properly configured
+2. Message handler not registered
+3. Queue resource not found
+4. SQS/RabbitMQ connection failed
+5. Message processing timeout
+
+Solution:
+Check queue configuration and message handler registration.
+
+Docs: https://github.com/forattini-dev/s3db.js/blob/main/docs/plugins/queue.md
+`.trim();
+    }
+
+    super(message, { ...rest, queueName, operation, messageId, description });
+  }
+}
+
+export default QueueError;

package/src/plugins/replicator.errors.js

@@ -0,0 +1,46 @@
+import { S3dbError } from '../errors.js';
+
+/**
+ * ReplicationError - Errors related to replication operations
+ *
+ * Used for replicator operations including:
+ * - Replicator initialization and setup
+ * - Data replication to target systems
+ * - Resource mapping and transformation
+ * - Connection management
+ * - Batch replication operations
+ *
+ * @extends S3dbError
+ */
+export class ReplicationError extends S3dbError {
+  constructor(message, details = {}) {
+    const { replicatorClass = 'unknown', operation = 'unknown', resourceName, ...rest } = details;
+
+    let description = details.description;
+    if (!description) {
+      description = `
+Replication Operation Error
+
+Replicator: ${replicatorClass}
+Operation: ${operation}
+${resourceName ? `Resource: ${resourceName}` : ''}
+
+Common causes:
+1. Invalid replicator configuration
+2. Target system not accessible
+3. Resource not configured for replication
+4. Invalid operation type
+5. Transformation function errors
+
+Solution:
+Check replicator configuration and ensure target system is accessible.
+
+Docs: https://github.com/forattini-dev/s3db.js/blob/main/docs/plugins/replicator.md
+`.trim();
+    }
+
+    super(message, { ...rest, replicatorClass, operation, resourceName, description });
+  }
+}
+
+export default ReplicationError;

package/src/plugins/replicator.plugin.js

@@ -1,6 +1,7 @@
 import Plugin from "./plugin.class.js";
 import tryFn from "../concerns/try-fn.js";
 import { createReplicator, validateReplicatorConfig } from "./replicators/index.js";
+import { ReplicationError } from "./replicator.errors.js";
 
 function normalizeResourceName(name) {
   return typeof name === 'string' ? name.trim().toLowerCase() : name;
@@ -120,12 +121,40 @@ export class ReplicatorPlugin extends Plugin {
     super();
     // Validation for config tests
     if (!options.replicators || !Array.isArray(options.replicators)) {
-      throw new Error('ReplicatorPlugin: replicators array is required');
+      throw new ReplicationError('ReplicatorPlugin requires replicators array', {
+        operation: 'constructor',
+        pluginName: 'ReplicatorPlugin',
+        providedOptions: Object.keys(options),
+        suggestion: 'Provide replicators array: new ReplicatorPlugin({ replicators: [{ driver: "s3db", resources: [...] }] })'
+      });
     }
     for (const rep of options.replicators) {
-      if (!rep.driver) throw new Error('ReplicatorPlugin: each replicator must have a driver');
-      if (!rep.resources || typeof rep.resources !== 'object') throw new Error('ReplicatorPlugin: each replicator must have resources config');
-      if (Object.keys(rep.resources).length === 0) throw new Error('ReplicatorPlugin: each replicator must have at least one resource configured');
+      if (!rep.driver) {
+        throw new ReplicationError('Each replicator must have a driver', {
+          operation: 'constructor',
+          pluginName: 'ReplicatorPlugin',
+          replicatorConfig: rep,
+          suggestion: 'Each replicator entry must specify a driver: { driver: "s3db", resources: {...} }'
+        });
+      }
+      if (!rep.resources || typeof rep.resources !== 'object') {
+        throw new ReplicationError('Each replicator must have resources config', {
+          operation: 'constructor',
+          pluginName: 'ReplicatorPlugin',
+          driver: rep.driver,
+          replicatorConfig: rep,
+          suggestion: 'Provide resources as object or array: { driver: "s3db", resources: ["users"] } or { resources: { users: "people" } }'
+        });
+      }
+      if (Object.keys(rep.resources).length === 0) {
+        throw new ReplicationError('Each replicator must have at least one resource configured', {
+          operation: 'constructor',
+          pluginName: 'ReplicatorPlugin',
+          driver: rep.driver,
+          replicatorConfig: rep,
+          suggestion: 'Add at least one resource to replicate: { driver: "s3db", resources: ["users"] }'
+        });
+      }
     }
     
     this.config = {
@@ -657,7 +686,13 @@ export class ReplicatorPlugin extends Plugin {
   async syncAllData(replicatorId) {
     const replicator = this.replicators.find(r => r.id === replicatorId);
     if (!replicator) {
-      throw new Error(`Replicator not found: ${replicatorId}`);
+      throw new ReplicationError('Replicator not found', {
+        operation: 'syncAllData',
+        pluginName: 'ReplicatorPlugin',
+        replicatorId,
+        availableReplicators: this.replicators.map(r => r.id),
+        suggestion: 'Check replicator ID or use getReplicatorStats() to list available replicators'
+      });
     }
 
     this.stats.lastSync = new Date().toISOString();

package/src/plugins/replicators/base-replicator.class.js

@@ -1,4 +1,5 @@
 import EventEmitter from 'events';
+import { ReplicationError } from '../replicator.errors.js';
 
 /**
  * Base class for all replicator drivers
@@ -31,7 +32,12 @@ export class BaseReplicator extends EventEmitter {
    * @returns {Promise<Object>} replicator result
    */
   async replicate(resourceName, operation, data, id) {
-    throw new Error(`replicate() method must be implemented by ${this.name}`);
+    throw new ReplicationError('replicate() method must be implemented by subclass', {
+      operation: 'replicate',
+      replicatorClass: this.name,
+      resourceName,
+      suggestion: 'Extend BaseReplicator and implement the replicate() method'
+    });
   }
 
   /**
@@ -41,7 +47,13 @@ export class BaseReplicator extends EventEmitter {
    * @returns {Promise<Object>} Batch replicator result
    */
   async replicateBatch(resourceName, records) {
-    throw new Error(`replicateBatch() method must be implemented by ${this.name}`);
+    throw new ReplicationError('replicateBatch() method must be implemented by subclass', {
+      operation: 'replicateBatch',
+      replicatorClass: this.name,
+      resourceName,
+      batchSize: records?.length,
+      suggestion: 'Extend BaseReplicator and implement the replicateBatch() method'
+    });
   }
 
   /**
@@ -49,7 +61,11 @@ export class BaseReplicator extends EventEmitter {
    * @returns {Promise<boolean>} True if connection is successful
    */
   async testConnection() {
-    throw new Error(`testConnection() method must be implemented by ${this.name}`);
+    throw new ReplicationError('testConnection() method must be implemented by subclass', {
+      operation: 'testConnection',
+      replicatorClass: this.name,
+      suggestion: 'Extend BaseReplicator and implement the testConnection() method'
+    });
   }
 
   /**

package/src/plugins/replicators/index.js

@@ -3,6 +3,7 @@ import BigqueryReplicator from './bigquery-replicator.class.js';
 import PostgresReplicator from './postgres-replicator.class.js';
 import S3dbReplicator from './s3db-replicator.class.js';
 import SqsReplicator from './sqs-replicator.class.js';
+import { ReplicationError } from '../replicator.errors.js';
 
 export { BaseReplicator, BigqueryReplicator, PostgresReplicator, S3dbReplicator, SqsReplicator };
 
@@ -24,11 +25,16 @@ export const REPLICATOR_DRIVERS = {
  */
 export function createReplicator(driver, config = {}, resources = [], client = null) {
   const ReplicatorClass = REPLICATOR_DRIVERS[driver];
-  
+
   if (!ReplicatorClass) {
-    throw new Error(`Unknown replicator driver: ${driver}. Available drivers: ${Object.keys(REPLICATOR_DRIVERS).join(', ')}`);
+    throw new ReplicationError(`Unknown replicator driver: ${driver}`, {
+      operation: 'createReplicator',
+      driver,
+      availableDrivers: Object.keys(REPLICATOR_DRIVERS),
+      suggestion: `Use one of the available drivers: ${Object.keys(REPLICATOR_DRIVERS).join(', ')}`
+    });
   }
-  
+
   return new ReplicatorClass(config, resources, client);
 }