s3db.js 9.3.0 → 10.0.0

package/README.md

@@ -329,11 +329,65 @@ const s3db = new S3db({
 });
 ```
 
-#### 3. S3-Compatible Services (MinIO, etc.)
+#### 3. MinIO (Self-hosted S3)
 ```javascript
+// MinIO running locally (note: http:// protocol and port)
 const s3db = new S3db({
-  connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
-  endpoint: "http://localhost:9000"
+  connectionString: "http://minioadmin:minioadmin@localhost:9000/mybucket/databases/myapp"
+});
+
+// MinIO on custom server
+const s3db = new S3db({
+  connectionString: "http://ACCESS_KEY:SECRET_KEY@minio.example.com:9000/BUCKET_NAME/databases/myapp"
+});
+```
+
+#### 4. Digital Ocean Spaces (SaaS)
+```javascript
+// Digital Ocean Spaces (NYC3 datacenter) - uses https:// as it's a public service
+const s3db = new S3db({
+  connectionString: "https://SPACES_KEY:SPACES_SECRET@nyc3.digitaloceanspaces.com/SPACE_NAME/databases/myapp"
+});
+
+// Other regions available: sfo3, ams3, sgp1, fra1, syd1
+const s3db = new S3db({
+  connectionString: "https://SPACES_KEY:SPACES_SECRET@sgp1.digitaloceanspaces.com/SPACE_NAME/databases/myapp"
+});
+```
+
+#### 5. LocalStack (Local AWS testing)
+```javascript
+// LocalStack for local development/testing (http:// with port 4566)
+const s3db = new S3db({
+  connectionString: "http://test:test@localhost:4566/mybucket/databases/myapp"
+});
+
+// LocalStack in Docker container
+const s3db = new S3db({
+  connectionString: "http://test:test@localstack:4566/mybucket/databases/myapp"
+});
+```
+
+#### 6. Other S3-Compatible Services
+```javascript
+// Backblaze B2 (SaaS - uses https://)
+const s3db = new S3db({
+  connectionString: "https://KEY_ID:APPLICATION_KEY@s3.us-west-002.backblazeb2.com/BUCKET_NAME/databases/myapp"
+});
+
+// Wasabi (SaaS - uses https://)
+const s3db = new S3db({
+  connectionString: "https://ACCESS_KEY:SECRET_KEY@s3.wasabisys.com/BUCKET_NAME/databases/myapp"
+});
+
+// Cloudflare R2 (SaaS - uses https://)
+const s3db = new S3db({
+  connectionString: "https://ACCESS_KEY:SECRET_KEY@ACCOUNT_ID.r2.cloudflarestorage.com/BUCKET_NAME/databases/myapp"
+});
+
+// Self-hosted Ceph with S3 gateway (http:// with custom port)
+const s3db = new S3db({
+  connectionString: "http://ACCESS_KEY:SECRET_KEY@ceph.internal:7480/BUCKET_NAME/databases/myapp"
 });
 ```
 
@@ -732,16 +786,20 @@ await users.insert({ name: "John", email: "john@example.com" });
 
 #### Available Plugins
 
-- **💾 Cache Plugin** - Intelligent caching (memory/S3) for performance
-- **💰 Costs Plugin** - Real-time AWS S3 cost tracking  
-- **🔍 FullText Plugin** - Advanced search with automatic indexing
-- **📊 Metrics Plugin** - Performance monitoring and analytics
-- **🔄 Replicator Plugin** - Multi-target replication (S3DB, SQS, BigQuery, PostgreSQL)
-- **📝 Audit Plugin** - Comprehensive audit logging for compliance
-- **📬 Queue Consumer Plugin** - Message consumption from SQS/RabbitMQ
-
-**📖 For detailed documentation, configuration options, and advanced examples, see:**
-**[📋 Complete Plugin Documentation](https://github.com/forattini-dev/s3db.js/blob/main/PLUGINS.md)**
+- **💾 [Cache Plugin](./docs/plugins/cache.md)** - Intelligent caching (memory/S3) for performance
+- **💰 [Costs Plugin](./docs/plugins/costs.md)** - Real-time AWS S3 cost tracking  
+- **🔍 [FullText Plugin](./docs/plugins/fulltext.md)** - Advanced search with automatic indexing
+- **📊 [Metrics Plugin](./docs/plugins/metrics.md)** - Performance monitoring and analytics
+- **🔄 [Replicator Plugin](./docs/plugins/replicator.md)** - Multi-target replication (S3DB, SQS, BigQuery, PostgreSQL)
+- **📝 [Audit Plugin](./docs/plugins/audit.md)** - Comprehensive audit logging for compliance
+- **📬 [Queue Consumer Plugin](./docs/plugins/queue-consumer.md)** - Message consumption from SQS/RabbitMQ
+- **📈 [Eventual Consistency Plugin](./docs/plugins/eventual-consistency.md)** - Event sourcing for numeric fields
+- **📅 [Scheduler Plugin](./docs/plugins/scheduler.md)** - Task scheduling and automation
+- **🔄 [State Machine Plugin](./docs/plugins/state-machine.md)** - State management and transitions
+- **💾 [Backup Plugin](./docs/plugins/backup.md)** - Backup and restore functionality
+
+**📖 For complete plugin documentation and overview:**
+**[📋 Plugin Documentation Index](./docs/plugins/README.md)**
 
 ### 🎛️ Resource Behaviors
 

package/dist/s3db.cjs.js

@@ -423,8 +423,14 @@ function mapAwsError(err, context = {}) {
     suggestion = "Check if the object metadata is present and valid.";
     return new MissingMetadata({ ...context, original: err, metadata, commandName, commandInput, suggestion });
   }
-  suggestion = "Check the error details and AWS documentation.";
-  return new UnknownError("Unknown error", { ...context, original: err, metadata, commandName, commandInput, suggestion });
+  const errorDetails = [
+    `Unknown error: ${err.message || err.toString()}`,
+    err.code && `Code: ${err.code}`,
+    err.statusCode && `Status: ${err.statusCode}`,
+    err.stack && `Stack: ${err.stack.split("\n")[0]}`
+  ].filter(Boolean).join(" | ");
+  suggestion = `Check the error details and AWS documentation. Original error: ${err.message || err.toString()}`;
+  return new UnknownError(errorDetails, { ...context, original: err, metadata, commandName, commandInput, suggestion });
 }
 class ConnectionStringError extends S3dbError {
   constructor(message, details = {}) {
@@ -1903,7 +1909,7 @@ class BackupPlugin extends Plugin {
       include: options.include || null,
       exclude: options.exclude || [],
       backupMetadataResource: options.backupMetadataResource || "backup_metadata",
-      tempDir: options.tempDir || "./tmp/backups",
+      tempDir: options.tempDir || "/tmp/s3db/backups",
       verbose: options.verbose || false,
       // Hooks
       onBackupStart: options.onBackupStart || null,
@@ -4050,6 +4056,457 @@ const CostsPlugin = {
   }
 };
 
+class EventualConsistencyPlugin extends Plugin {
+  constructor(options = {}) {
+    super(options);
+    if (!options.resource) {
+      throw new Error("EventualConsistencyPlugin requires 'resource' option");
+    }
+    if (!options.field) {
+      throw new Error("EventualConsistencyPlugin requires 'field' option");
+    }
+    this.config = {
+      resource: options.resource,
+      field: options.field,
+      cohort: {
+        interval: options.cohort?.interval || "24h",
+        timezone: options.cohort?.timezone || "UTC",
+        ...options.cohort
+      },
+      reducer: options.reducer || ((transactions) => {
+        let baseValue = 0;
+        for (const t of transactions) {
+          if (t.operation === "set") {
+            baseValue = t.value;
+          } else if (t.operation === "add") {
+            baseValue += t.value;
+          } else if (t.operation === "sub") {
+            baseValue -= t.value;
+          }
+        }
+        return baseValue;
+      }),
+      consolidationInterval: options.consolidationInterval || 36e5,
+      // 1 hour default
+      autoConsolidate: options.autoConsolidate !== false,
+      batchTransactions: options.batchTransactions || false,
+      batchSize: options.batchSize || 100,
+      mode: options.mode || "async",
+      // 'async' or 'sync'
+      ...options
+    };
+    this.transactionResource = null;
+    this.targetResource = null;
+    this.consolidationTimer = null;
+    this.pendingTransactions = /* @__PURE__ */ new Map();
+  }
+  async onSetup() {
+    this.targetResource = this.database.resources[this.config.resource];
+    if (!this.targetResource) {
+      this.deferredSetup = true;
+      this.watchForResource();
+      return;
+    }
+    await this.completeSetup();
+  }
+  watchForResource() {
+    const hookCallback = async ({ resource, config }) => {
+      if (config.name === this.config.resource && this.deferredSetup) {
+        this.targetResource = resource;
+        this.deferredSetup = false;
+        await this.completeSetup();
+      }
+    };
+    this.database.addHook("afterCreateResource", hookCallback);
+  }
+  async completeSetup() {
+    if (!this.targetResource) return;
+    const transactionResourceName = `${this.config.resource}_transactions_${this.config.field}`;
+    const partitionConfig = this.createPartitionConfig();
+    const [ok, err, transactionResource] = await tryFn(
+      () => this.database.createResource({
+        name: transactionResourceName,
+        attributes: {
+          id: "string|required",
+          originalId: "string|required",
+          field: "string|required",
+          value: "number|required",
+          operation: "string|required",
+          // 'set', 'add', or 'sub'
+          timestamp: "string|required",
+          cohortDate: "string|required",
+          // For partitioning
+          cohortMonth: "string|optional",
+          // For monthly partitioning
+          source: "string|optional",
+          applied: "boolean|optional"
+          // Track if transaction was applied
+        },
+        behavior: "body-overflow",
+        timestamps: true,
+        partitions: partitionConfig,
+        asyncPartitions: true
+        // Use async partitions for better performance
+      })
+    );
+    if (!ok && !this.database.resources[transactionResourceName]) {
+      throw new Error(`Failed to create transaction resource: ${err?.message}`);
+    }
+    this.transactionResource = ok ? transactionResource : this.database.resources[transactionResourceName];
+    this.addHelperMethods();
+    if (this.config.autoConsolidate) {
+      this.startConsolidationTimer();
+    }
+  }
+  async onStart() {
+    if (this.deferredSetup) {
+      return;
+    }
+    this.emit("eventual-consistency.started", {
+      resource: this.config.resource,
+      field: this.config.field,
+      cohort: this.config.cohort
+    });
+  }
+  async onStop() {
+    if (this.consolidationTimer) {
+      clearInterval(this.consolidationTimer);
+      this.consolidationTimer = null;
+    }
+    await this.flushPendingTransactions();
+    this.emit("eventual-consistency.stopped", {
+      resource: this.config.resource,
+      field: this.config.field
+    });
+  }
+  createPartitionConfig() {
+    const partitions = {
+      byDay: {
+        fields: {
+          cohortDate: "string"
+        }
+      },
+      byMonth: {
+        fields: {
+          cohortMonth: "string"
+        }
+      }
+    };
+    return partitions;
+  }
+  addHelperMethods() {
+    const resource = this.targetResource;
+    const defaultField = this.config.field;
+    const plugin = this;
+    if (!resource._eventualConsistencyPlugins) {
+      resource._eventualConsistencyPlugins = {};
+    }
+    resource._eventualConsistencyPlugins[defaultField] = plugin;
+    resource.set = async (id, fieldOrValue, value) => {
+      const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
+      if (hasMultipleFields && value === void 0) {
+        throw new Error(`Multiple fields have eventual consistency. Please specify the field: set(id, field, value)`);
+      }
+      const field = value !== void 0 ? fieldOrValue : defaultField;
+      const actualValue = value !== void 0 ? value : fieldOrValue;
+      const fieldPlugin = resource._eventualConsistencyPlugins[field];
+      if (!fieldPlugin) {
+        throw new Error(`No eventual consistency plugin found for field "${field}"`);
+      }
+      await fieldPlugin.createTransaction({
+        originalId: id,
+        operation: "set",
+        value: actualValue,
+        source: "set"
+      });
+      if (fieldPlugin.config.mode === "sync") {
+        const consolidatedValue = await fieldPlugin.consolidateRecord(id);
+        await resource.update(id, {
+          [field]: consolidatedValue
+        });
+        return consolidatedValue;
+      }
+      return actualValue;
+    };
+    resource.add = async (id, fieldOrAmount, amount) => {
+      const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
+      if (hasMultipleFields && amount === void 0) {
+        throw new Error(`Multiple fields have eventual consistency. Please specify the field: add(id, field, amount)`);
+      }
+      const field = amount !== void 0 ? fieldOrAmount : defaultField;
+      const actualAmount = amount !== void 0 ? amount : fieldOrAmount;
+      const fieldPlugin = resource._eventualConsistencyPlugins[field];
+      if (!fieldPlugin) {
+        throw new Error(`No eventual consistency plugin found for field "${field}"`);
+      }
+      await fieldPlugin.createTransaction({
+        originalId: id,
+        operation: "add",
+        value: actualAmount,
+        source: "add"
+      });
+      if (fieldPlugin.config.mode === "sync") {
+        const consolidatedValue = await fieldPlugin.consolidateRecord(id);
+        await resource.update(id, {
+          [field]: consolidatedValue
+        });
+        return consolidatedValue;
+      }
+      const currentValue = await fieldPlugin.getConsolidatedValue(id);
+      return currentValue + actualAmount;
+    };
+    resource.sub = async (id, fieldOrAmount, amount) => {
+      const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
+      if (hasMultipleFields && amount === void 0) {
+        throw new Error(`Multiple fields have eventual consistency. Please specify the field: sub(id, field, amount)`);
+      }
+      const field = amount !== void 0 ? fieldOrAmount : defaultField;
+      const actualAmount = amount !== void 0 ? amount : fieldOrAmount;
+      const fieldPlugin = resource._eventualConsistencyPlugins[field];
+      if (!fieldPlugin) {
+        throw new Error(`No eventual consistency plugin found for field "${field}"`);
+      }
+      await fieldPlugin.createTransaction({
+        originalId: id,
+        operation: "sub",
+        value: actualAmount,
+        source: "sub"
+      });
+      if (fieldPlugin.config.mode === "sync") {
+        const consolidatedValue = await fieldPlugin.consolidateRecord(id);
+        await resource.update(id, {
+          [field]: consolidatedValue
+        });
+        return consolidatedValue;
+      }
+      const currentValue = await fieldPlugin.getConsolidatedValue(id);
+      return currentValue - actualAmount;
+    };
+    resource.consolidate = async (id, field) => {
+      const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
+      if (hasMultipleFields && !field) {
+        throw new Error(`Multiple fields have eventual consistency. Please specify the field: consolidate(id, field)`);
+      }
+      const actualField = field || defaultField;
+      const fieldPlugin = resource._eventualConsistencyPlugins[actualField];
+      if (!fieldPlugin) {
+        throw new Error(`No eventual consistency plugin found for field "${actualField}"`);
+      }
+      return await fieldPlugin.consolidateRecord(id);
+    };
+    resource.getConsolidatedValue = async (id, fieldOrOptions, options) => {
+      if (typeof fieldOrOptions === "string") {
+        const field = fieldOrOptions;
+        const fieldPlugin = resource._eventualConsistencyPlugins[field] || plugin;
+        return await fieldPlugin.getConsolidatedValue(id, options || {});
+      } else {
+        return await plugin.getConsolidatedValue(id, fieldOrOptions || {});
+      }
+    };
+  }
+  async createTransaction(data) {
+    const now = /* @__PURE__ */ new Date();
+    const cohortInfo = this.getCohortInfo(now);
+    const transaction = {
+      id: `txn-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
+      originalId: data.originalId,
+      field: this.config.field,
+      value: data.value || 0,
+      operation: data.operation || "set",
+      timestamp: now.toISOString(),
+      cohortDate: cohortInfo.date,
+      cohortMonth: cohortInfo.month,
+      source: data.source || "unknown",
+      applied: false
+    };
+    if (this.config.batchTransactions) {
+      this.pendingTransactions.set(transaction.id, transaction);
+      if (this.pendingTransactions.size >= this.config.batchSize) {
+        await this.flushPendingTransactions();
+      }
+    } else {
+      await this.transactionResource.insert(transaction);
+    }
+    return transaction;
+  }
+  async flushPendingTransactions() {
+    if (this.pendingTransactions.size === 0) return;
+    const transactions = Array.from(this.pendingTransactions.values());
+    this.pendingTransactions.clear();
+    for (const transaction of transactions) {
+      await this.transactionResource.insert(transaction);
+    }
+  }
+  getCohortInfo(date) {
+    const tz = this.config.cohort.timezone;
+    const offset = this.getTimezoneOffset(tz);
+    const localDate = new Date(date.getTime() + offset);
+    const year = localDate.getFullYear();
+    const month = String(localDate.getMonth() + 1).padStart(2, "0");
+    const day = String(localDate.getDate()).padStart(2, "0");
+    return {
+      date: `${year}-${month}-${day}`,
+      month: `${year}-${month}`
+    };
+  }
+  getTimezoneOffset(timezone) {
+    const offsets = {
+      "UTC": 0,
+      "America/New_York": -5 * 36e5,
+      "America/Chicago": -6 * 36e5,
+      "America/Denver": -7 * 36e5,
+      "America/Los_Angeles": -8 * 36e5,
+      "America/Sao_Paulo": -3 * 36e5,
+      "Europe/London": 0,
+      "Europe/Paris": 1 * 36e5,
+      "Europe/Berlin": 1 * 36e5,
+      "Asia/Tokyo": 9 * 36e5,
+      "Asia/Shanghai": 8 * 36e5,
+      "Australia/Sydney": 10 * 36e5
+    };
+    return offsets[timezone] || 0;
+  }
+  startConsolidationTimer() {
+    const interval = this.config.consolidationInterval;
+    this.consolidationTimer = setInterval(async () => {
+      await this.runConsolidation();
+    }, interval);
+  }
+  async runConsolidation() {
+    try {
+      const [ok, err, transactions] = await tryFn(
+        () => this.transactionResource.query({
+          applied: false
+        })
+      );
+      if (!ok) {
+        console.error("Consolidation failed to query transactions:", err);
+        return;
+      }
+      const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
+      for (const id of uniqueIds) {
+        await this.consolidateRecord(id);
+      }
+      this.emit("eventual-consistency.consolidated", {
+        resource: this.config.resource,
+        field: this.config.field,
+        recordCount: uniqueIds.length
+      });
+    } catch (error) {
+      console.error("Consolidation error:", error);
+      this.emit("eventual-consistency.consolidation-error", error);
+    }
+  }
+  async consolidateRecord(originalId) {
+    const [recordOk, recordErr, record] = await tryFn(
+      () => this.targetResource.get(originalId)
+    );
+    const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
+    const [ok, err, transactions] = await tryFn(
+      () => this.transactionResource.query({
+        originalId,
+        applied: false
+      })
+    );
+    if (!ok || !transactions || transactions.length === 0) {
+      return currentValue;
+    }
+    transactions.sort(
+      (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
+    );
+    const hasSetOperation = transactions.some((t) => t.operation === "set");
+    if (currentValue !== 0 && !hasSetOperation) {
+      transactions.unshift({
+        id: "__synthetic__",
+        // Synthetic ID that we'll skip when marking as applied
+        operation: "set",
+        value: currentValue,
+        timestamp: (/* @__PURE__ */ new Date(0)).toISOString()
+        // Very old timestamp to ensure it's first
+      });
+    }
+    const consolidatedValue = this.config.reducer(transactions);
+    const [updateOk, updateErr] = await tryFn(
+      () => this.targetResource.update(originalId, {
+        [this.config.field]: consolidatedValue
+      })
+    );
+    if (updateOk) {
+      for (const txn of transactions) {
+        if (txn.id !== "__synthetic__") {
+          await this.transactionResource.update(txn.id, {
+            applied: true
+          });
+        }
+      }
+    }
+    return consolidatedValue;
+  }
+  async getConsolidatedValue(originalId, options = {}) {
+    const includeApplied = options.includeApplied || false;
+    const startDate = options.startDate;
+    const endDate = options.endDate;
+    const query = { originalId };
+    if (!includeApplied) {
+      query.applied = false;
+    }
+    const [ok, err, transactions] = await tryFn(
+      () => this.transactionResource.query(query)
+    );
+    if (!ok || !transactions || transactions.length === 0) {
+      const [recordOk, recordErr, record] = await tryFn(
+        () => this.targetResource.get(originalId)
+      );
+      if (recordOk && record) {
+        return record[this.config.field] || 0;
+      }
+      return 0;
+    }
+    let filtered = transactions;
+    if (startDate || endDate) {
+      filtered = transactions.filter((t) => {
+        const timestamp = new Date(t.timestamp);
+        if (startDate && timestamp < new Date(startDate)) return false;
+        if (endDate && timestamp > new Date(endDate)) return false;
+        return true;
+      });
+    }
+    filtered.sort(
+      (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
+    );
+    return this.config.reducer(filtered);
+  }
+  // Helper method to get cohort statistics
+  async getCohortStats(cohortDate) {
+    const [ok, err, transactions] = await tryFn(
+      () => this.transactionResource.query({
+        cohortDate
+      })
+    );
+    if (!ok) return null;
+    const stats = {
+      date: cohortDate,
+      transactionCount: transactions.length,
+      totalValue: 0,
+      byOperation: { set: 0, add: 0, sub: 0 },
+      byOriginalId: {}
+    };
+    for (const txn of transactions) {
+      stats.totalValue += txn.value || 0;
+      stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
+      if (!stats.byOriginalId[txn.originalId]) {
+        stats.byOriginalId[txn.originalId] = {
+          count: 0,
+          value: 0
+        };
+      }
+      stats.byOriginalId[txn.originalId].count++;
+      stats.byOriginalId[txn.originalId].value += txn.value || 0;
+    }
+    return stats;
+  }
+}
+
 class FullTextPlugin extends Plugin {
   constructor(options = {}) {
     super();
@@ -5834,10 +6291,10 @@ class Client extends EventEmitter {
       // Enabled for better performance
       keepAliveMsecs: 1e3,
       // 1 second keep-alive
-      maxSockets: 50,
-      // Balanced for most applications
-      maxFreeSockets: 10,
-      // Good connection reuse
+      maxSockets: httpClientOptions.maxSockets || 500,
+      // High concurrency support
+      maxFreeSockets: httpClientOptions.maxFreeSockets || 100,
+      // Better connection reuse
       timeout: 6e4,
       // 60 second timeout
       ...httpClientOptions
@@ -9700,7 +10157,7 @@ class Database extends EventEmitter {
     this.id = idGenerator(7);
     this.version = "1";
     this.s3dbVersion = (() => {
-      const [ok, err, version] = tryFn(() => true ? "9.3.0" : "latest");
+      const [ok, err, version] = tryFn(() => true ? "10.0.0" : "latest");
       return ok ? version : "latest";
     })();
     this.resources = {};
@@ -12785,6 +13242,7 @@ exports.Database = Database;
 exports.DatabaseError = DatabaseError;
 exports.EncryptionError = EncryptionError;
 exports.ErrorMap = ErrorMap;
+exports.EventualConsistencyPlugin = EventualConsistencyPlugin;
 exports.FullTextPlugin = FullTextPlugin;
 exports.InvalidResourceItem = InvalidResourceItem;
 exports.MetricsPlugin = MetricsPlugin;