s3db.js 11.3.2 → 12.0.1

package/src/plugins/concerns/prometheus-formatter.js

@@ -0,0 +1,255 @@
+/**
+ * Prometheus Formatter - Format s3db.js metrics to Prometheus text-based format
+ *
+ * Generates metrics in Prometheus exposition format:
+ * https://prometheus.io/docs/instrumenting/exposition_formats/
+ */
+
+/**
+ * Sanitize label value for Prometheus
+ * - Replace invalid characters with underscores
+ * - Escape special characters
+ * @param {string} value - Label value
+ * @returns {string} Sanitized value
+ */
+function sanitizeLabel(value) {
+  if (typeof value !== 'string') {
+    value = String(value);
+  }
+
+  // Escape backslashes and quotes
+  return value
+    .replace(/\\/g, '\\\\')
+    .replace(/"/g, '\\"')
+    .replace(/\n/g, '\\n');
+}
+
+/**
+ * Sanitize metric name for Prometheus
+ * - Only alphanumeric and underscores allowed
+ * - Must not start with digit
+ * @param {string} name - Metric name
+ * @returns {string} Sanitized name
+ */
+function sanitizeMetricName(name) {
+  // Replace invalid characters with underscores
+  let sanitized = name.replace(/[^a-zA-Z0-9_]/g, '_');
+
+  // Ensure doesn't start with digit
+  if (/^\d/.test(sanitized)) {
+    sanitized = '_' + sanitized;
+  }
+
+  return sanitized;
+}
+
+/**
+ * Format labels for Prometheus metric line
+ * @param {Object} labels - Label key-value pairs
+ * @returns {string} Formatted labels string
+ */
+function formatLabels(labels) {
+  if (!labels || Object.keys(labels).length === 0) {
+    return '';
+  }
+
+  const labelPairs = Object.entries(labels)
+    .map(([key, value]) => `${key}="${sanitizeLabel(value)}"`)
+    .join(',');
+
+  return `{${labelPairs}}`;
+}
+
+/**
+ * Format a single Prometheus metric
+ * @param {string} name - Metric name
+ * @param {string} type - Metric type (counter, gauge, histogram, summary)
+ * @param {string} help - Help text
+ * @param {Array<{labels: Object, value: number}>} values - Metric values with labels
+ * @returns {string} Formatted metric lines
+ */
+function formatMetric(name, type, help, values) {
+  const lines = [];
+
+  // HELP line
+  lines.push(`# HELP ${name} ${help}`);
+
+  // TYPE line
+  lines.push(`# TYPE ${name} ${type}`);
+
+  // Value lines
+  for (const { labels, value } of values) {
+    const labelsStr = formatLabels(labels);
+    lines.push(`${name}${labelsStr} ${value}`);
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Format all metrics from MetricsPlugin to Prometheus format
+ * @param {MetricsPlugin} metricsPlugin - Instance of MetricsPlugin
+ * @returns {string} Complete Prometheus metrics text
+ */
+export function formatPrometheusMetrics(metricsPlugin) {
+  const lines = [];
+  const metrics = metricsPlugin.metrics;
+
+  // 1. Operations Total (counter)
+  const operationsTotalValues = [];
+
+  // Global operations
+  for (const [operation, data] of Object.entries(metrics.operations)) {
+    if (data.count > 0) {
+      operationsTotalValues.push({
+        labels: { operation, resource: '_global' },
+        value: data.count
+      });
+    }
+  }
+
+  // Resource-specific operations
+  for (const [resourceName, operations] of Object.entries(metrics.resources)) {
+    for (const [operation, data] of Object.entries(operations)) {
+      if (data.count > 0) {
+        operationsTotalValues.push({
+          labels: { operation, resource: sanitizeMetricName(resourceName) },
+          value: data.count
+        });
+      }
+    }
+  }
+
+  if (operationsTotalValues.length > 0) {
+    lines.push(formatMetric(
+      's3db_operations_total',
+      'counter',
+      'Total number of operations by type and resource',
+      operationsTotalValues
+    ));
+    lines.push('');
+  }
+
+  // 2. Operation Duration (gauge - average)
+  const durationValues = [];
+
+  // Global operations
+  for (const [operation, data] of Object.entries(metrics.operations)) {
+    if (data.count > 0) {
+      const avgSeconds = (data.totalTime / data.count) / 1000; // Convert ms to seconds
+      durationValues.push({
+        labels: { operation, resource: '_global' },
+        value: avgSeconds.toFixed(6)
+      });
+    }
+  }
+
+  // Resource-specific operations
+  for (const [resourceName, operations] of Object.entries(metrics.resources)) {
+    for (const [operation, data] of Object.entries(operations)) {
+      if (data.count > 0) {
+        const avgSeconds = (data.totalTime / data.count) / 1000; // Convert ms to seconds
+        durationValues.push({
+          labels: { operation, resource: sanitizeMetricName(resourceName) },
+          value: avgSeconds.toFixed(6)
+        });
+      }
+    }
+  }
+
+  if (durationValues.length > 0) {
+    lines.push(formatMetric(
+      's3db_operation_duration_seconds',
+      'gauge',
+      'Average operation duration in seconds',
+      durationValues
+    ));
+    lines.push('');
+  }
+
+  // 3. Operation Errors Total (counter)
+  const errorsValues = [];
+
+  // Global errors
+  for (const [operation, data] of Object.entries(metrics.operations)) {
+    if (data.errors > 0) {
+      errorsValues.push({
+        labels: { operation, resource: '_global' },
+        value: data.errors
+      });
+    }
+  }
+
+  // Resource-specific errors
+  for (const [resourceName, operations] of Object.entries(metrics.resources)) {
+    for (const [operation, data] of Object.entries(operations)) {
+      if (data.errors > 0) {
+        errorsValues.push({
+          labels: { operation, resource: sanitizeMetricName(resourceName) },
+          value: data.errors
+        });
+      }
+    }
+  }
+
+  if (errorsValues.length > 0) {
+    lines.push(formatMetric(
+      's3db_operation_errors_total',
+      'counter',
+      'Total number of operation errors',
+      errorsValues
+    ));
+    lines.push('');
+  }
+
+  // 4. Uptime (gauge)
+  const startTime = new Date(metrics.startTime);
+  const uptimeSeconds = (Date.now() - startTime.getTime()) / 1000;
+
+  lines.push(formatMetric(
+    's3db_uptime_seconds',
+    'gauge',
+    'Process uptime in seconds',
+    [{ labels: {}, value: uptimeSeconds.toFixed(2) }]
+  ));
+  lines.push('');
+
+  // 5. Resources Total (gauge)
+  const resourcesCount = Object.keys(metrics.resources).length;
+
+  lines.push(formatMetric(
+    's3db_resources_total',
+    'gauge',
+    'Total number of tracked resources',
+    [{ labels: {}, value: resourcesCount }]
+  ));
+  lines.push('');
+
+  // 6. Build Info (gauge - always 1)
+  const nodeVersion = process.version || 'unknown';
+  const s3dbVersion = '1.0.0'; // TODO: Get from package.json
+
+  lines.push(formatMetric(
+    's3db_info',
+    'gauge',
+    'Build and runtime information',
+    [{
+      labels: {
+        version: s3dbVersion,
+        node_version: nodeVersion
+      },
+      value: 1
+    }]
+  ));
+
+  // Join all lines with newline and ensure ends with newline
+  return lines.join('\n') + '\n';
+}
+
+export default {
+  formatPrometheusMetrics,
+  formatMetric,
+  sanitizeLabel,
+  sanitizeMetricName,
+  formatLabels
+};

package/src/plugins/consumers/rabbitmq-consumer.js

@@ -1,4 +1,5 @@
 import tryFn from "../../concerns/try-fn.js";
+import requirePluginDependency from "../concerns/plugin-dependencies.js";
 
 export class RabbitMqConsumer {
   constructor({ amqpUrl, queue, prefetch = 10, reconnectInterval = 2000, onMessage, onError, driver = 'rabbitmq' }) {
@@ -15,6 +16,9 @@ export class RabbitMqConsumer {
   }
 
   async start() {
+    // Validate plugin dependencies are installed
+    await requirePluginDependency('rabbitmq-consumer');
+
     this._stopped = false;
     await this._connect();
   }

package/src/plugins/consumers/sqs-consumer.js

@@ -1,4 +1,5 @@
 import tryFn from "../../concerns/try-fn.js";
+import requirePluginDependency from "../concerns/plugin-dependencies.js";
 // Remove static SDK import
 // import { SQSClient, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';
 
@@ -25,6 +26,9 @@ export class SqsConsumer {
   }
 
   async start() {
+    // Validate plugin dependencies are installed
+    await requirePluginDependency('sqs-consumer');
+
     // Carregar SDK dinamicamente
     const [ok, err, sdk] = await tryFn(() => import('@aws-sdk/client-sqs'));
     if (!ok) throw new Error('SqsConsumer: @aws-sdk/client-sqs is not installed. Please install it to use the SQS consumer.');

package/src/plugins/costs.plugin.js

@@ -1,14 +1,20 @@
 export const CostsPlugin = {
-  async setup (db) {
+  async setup (db, options = {}) {
     if (!db || !db.client) {
       return; // Handle null/invalid database gracefully
     }
 
     this.client = db.client
+    this.options = {
+      considerFreeTier: false,  // Flag to consider AWS free tier in calculations
+      region: 'us-east-1',      // AWS region for pricing (future use)
+      ...options
+    }
 
     this.map = {
       PutObjectCommand: 'put',
       GetObjectCommand: 'get',
+      CopyObjectCommand: 'copy',
       HeadObjectCommand: 'head',
       DeleteObjectCommand: 'delete',
       DeleteObjectsCommand: 'delete',
@@ -17,35 +23,79 @@ export const CostsPlugin = {
 
     this.costs = {
       total: 0,
-      prices: {
-        put: 0.005 / 1000,
-        copy: 0.005 / 1000,
-        list: 0.005 / 1000,
-        post: 0.005 / 1000,
-        get: 0.0004 / 1000,
-        select: 0.0004 / 1000,
-        delete: 0.0004 / 1000,
-        head: 0.0004 / 1000,
-      },
+
+      // === REQUESTS PRICING ===
       requests: {
+        prices: {
+          put: 0.005 / 1000,
+          copy: 0.005 / 1000,
+          list: 0.005 / 1000,
+          post: 0.005 / 1000,
+          get: 0.0004 / 1000,
+          select: 0.0004 / 1000,
+          delete: 0.0004 / 1000,
+          head: 0.0004 / 1000,
+        },
         total: 0,
-        put: 0,
-        post: 0,
-        copy: 0,
-        list: 0,
-        get: 0,
-        select: 0,
-        delete: 0,
-        head: 0,
+        counts: {
+          put: 0,
+          post: 0,
+          copy: 0,
+          list: 0,
+          get: 0,
+          select: 0,
+          delete: 0,
+          head: 0,
+        },
+        totalEvents: 0,
+        events: {
+          PutObjectCommand: 0,
+          GetObjectCommand: 0,
+          CopyObjectCommand: 0,
+          HeadObjectCommand: 0,
+          DeleteObjectCommand: 0,
+          DeleteObjectsCommand: 0,
+          ListObjectsV2Command: 0,
+        },
+        subtotal: 0,
       },
-      events: {
-        total: 0,
-        PutObjectCommand: 0,
-        GetObjectCommand: 0,
-        HeadObjectCommand: 0,
-        DeleteObjectCommand: 0,
-        DeleteObjectsCommand: 0,
-        ListObjectsV2Command: 0,
+
+      // === STORAGE PRICING ===
+      storage: {
+        totalBytes: 0,
+        totalGB: 0,
+        // Tiered pricing (S3 Standard - us-east-1)
+        tiers: [
+          { limit: 50 * 1024, pricePerGB: 0.023 },      // First 50 TB
+          { limit: 500 * 1024, pricePerGB: 0.022 },     // Next 450 TB
+          { limit: 999999999, pricePerGB: 0.021 }       // Over 500 TB (effectively unlimited)
+        ],
+        currentTier: 0,
+        subtotal: 0  // Monthly storage cost estimate
+      },
+
+      // === DATA TRANSFER PRICING ===
+      dataTransfer: {
+        // Upload (always free)
+        inBytes: 0,
+        inGB: 0,
+        inCost: 0,  // Always $0
+
+        // Download (charged with tiers)
+        outBytes: 0,
+        outGB: 0,
+        // Tiered pricing (out to internet)
+        tiers: [
+          { limit: 10 * 1024, pricePerGB: 0.09 },       // First 10 TB
+          { limit: 50 * 1024, pricePerGB: 0.085 },      // Next 40 TB
+          { limit: 150 * 1024, pricePerGB: 0.07 },      // Next 100 TB
+          { limit: 999999999, pricePerGB: 0.05 }        // Over 150 TB (effectively unlimited)
+        ],
+        // Free tier (100GB/month aggregated across AWS)
+        freeTierGB: 100,
+        freeTierUsed: 0,
+        currentTier: 0,
+        subtotal: 0  // Data transfer out cost
       }
     }
 
@@ -54,26 +104,192 @@ export const CostsPlugin = {
   
   async start () {
     if (this.client) {
-      this.client.on("command.response", (name) => this.addRequest(name, this.map[name]));
-      this.client.on("command.error", (name) => this.addRequest(name, this.map[name]));
+      this.client.on("command.response", (name, response, input) => this.addRequest(name, this.map[name], response, input));
+      this.client.on("command.error", (name, response, input) => this.addRequest(name, this.map[name], response, input));
     }
   },
 
-  addRequest (name, method) {
+  addRequest (name, method, response = {}, input = {}) {
     if (!method) return; // Skip if no mapping found
-    
-    this.costs.events[name]++;
-    this.costs.events.total++;
+
+    // Track request counts
+    this.costs.requests.totalEvents++;
     this.costs.requests.total++;
-    this.costs.requests[method]++;
-    this.costs.total += this.costs.prices[method];
+    this.costs.requests.events[name]++;
+    this.costs.requests.counts[method]++;
+
+    // Calculate request cost
+    const requestCost = this.costs.requests.prices[method];
+    this.costs.requests.subtotal += requestCost;
+
+    // Track storage and data transfer based on ContentLength
+    let contentLength = 0;
+
+    if (['put', 'post', 'copy'].includes(method)) {
+      // For uploads, get size from input Body (AWS SDK uses capital B)
+      const body = input.Body || input.body;
+      if (body) {
+        if (typeof body === 'string') {
+          contentLength = Buffer.byteLength(body, 'utf8');
+        } else if (Buffer.isBuffer(body)) {
+          contentLength = body.length;
+        } else if (body.length !== undefined) {
+          contentLength = body.length;
+        }
+      }
+
+      if (contentLength > 0) {
+        this.trackStorage(contentLength);
+        this.trackDataTransferIn(contentLength);
+      }
+    }
+
+    if (method === 'get') {
+      // For downloads, get size from response
+      contentLength = response?.httpResponse?.headers?.['content-length'] ||
+                     response?.ContentLength ||
+                     0;
 
+      if (contentLength > 0) {
+        this.trackDataTransferOut(contentLength);
+      }
+    }
+
+    // Mirror request-related counters to client.costs BEFORE updateTotal()
+    // (Storage and data transfer are mirrored in tracking methods)
     if (this.client && this.client.costs) {
-      this.client.costs.events[name]++;
-      this.client.costs.events.total++;
+      this.client.costs.requests.totalEvents++;
       this.client.costs.requests.total++;
-      this.client.costs.requests[method]++;      
-      this.client.costs.total += this.client.costs.prices[method];
+      this.client.costs.requests.events[name]++;
+      this.client.costs.requests.counts[method]++;
+      this.client.costs.requests.subtotal += requestCost;
+    }
+
+    // Update total cost (must be after mirroring request counters)
+    this.updateTotal();
+  },
+
+  trackStorage (bytes) {
+    this.costs.storage.totalBytes += bytes;
+    this.costs.storage.totalGB = this.costs.storage.totalBytes / (1024 * 1024 * 1024);
+    this.costs.storage.subtotal = this.calculateStorageCost(this.costs.storage);
+
+    // Mirror to client.costs
+    if (this.client && this.client.costs) {
+      this.client.costs.storage.totalBytes += bytes;
+      this.client.costs.storage.totalGB = this.client.costs.storage.totalBytes / (1024 * 1024 * 1024);
+      this.client.costs.storage.subtotal = this.calculateStorageCost(this.client.costs.storage);
+    }
+
+    // Update total cost
+    this.updateTotal();
+  },
+
+  trackDataTransferIn (bytes) {
+    this.costs.dataTransfer.inBytes += bytes;
+    this.costs.dataTransfer.inGB = this.costs.dataTransfer.inBytes / (1024 * 1024 * 1024);
+    // inCost is always $0
+
+    // Mirror to client.costs
+    if (this.client && this.client.costs) {
+      this.client.costs.dataTransfer.inBytes += bytes;
+      this.client.costs.dataTransfer.inGB = this.client.costs.dataTransfer.inBytes / (1024 * 1024 * 1024);
+    }
+
+    // Update total cost
+    this.updateTotal();
+  },
+
+  trackDataTransferOut (bytes) {
+    this.costs.dataTransfer.outBytes += bytes;
+    this.costs.dataTransfer.outGB = this.costs.dataTransfer.outBytes / (1024 * 1024 * 1024);
+    this.costs.dataTransfer.subtotal = this.calculateDataTransferCost(this.costs.dataTransfer);
+
+    // Mirror to client.costs
+    if (this.client && this.client.costs) {
+      this.client.costs.dataTransfer.outBytes += bytes;
+      this.client.costs.dataTransfer.outGB = this.client.costs.dataTransfer.outBytes / (1024 * 1024 * 1024);
+      this.client.costs.dataTransfer.subtotal = this.calculateDataTransferCost(this.client.costs.dataTransfer);
+    }
+
+    // Update total cost
+    this.updateTotal();
+  },
+
+  calculateStorageCost (storage) {
+    const totalGB = storage.totalGB;
+    let cost = 0;
+    let remaining = totalGB;
+
+    for (let i = 0; i < storage.tiers.length; i++) {
+      const tier = storage.tiers[i];
+      const prevLimit = i > 0 ? storage.tiers[i - 1].limit : 0;
+      const tierCapacity = tier.limit - prevLimit;
+
+      if (remaining <= 0) break;
+
+      const gbInTier = Math.min(remaining, tierCapacity);
+      cost += gbInTier * tier.pricePerGB;
+      remaining -= gbInTier;
+
+      if (remaining <= 0) {
+        storage.currentTier = i;
+        break;
+      }
+    }
+
+    return cost;
+  },
+
+  calculateDataTransferCost (dataTransfer) {
+    let totalGB = dataTransfer.outGB;
+    let cost = 0;
+
+    // Apply free tier if enabled
+    if (this.options && this.options.considerFreeTier) {
+      const freeTierRemaining = dataTransfer.freeTierGB - dataTransfer.freeTierUsed;
+
+      if (freeTierRemaining > 0 && totalGB > 0) {
+        const gbToDeduct = Math.min(totalGB, freeTierRemaining);
+        totalGB -= gbToDeduct;
+        dataTransfer.freeTierUsed += gbToDeduct;
+      }
+    }
+
+    // Calculate with tiers
+    let remaining = totalGB;
+    for (let i = 0; i < dataTransfer.tiers.length; i++) {
+      const tier = dataTransfer.tiers[i];
+      const prevLimit = i > 0 ? dataTransfer.tiers[i - 1].limit : 0;
+      const tierCapacity = tier.limit - prevLimit;
+
+      if (remaining <= 0) break;
+
+      const gbInTier = Math.min(remaining, tierCapacity);
+      cost += gbInTier * tier.pricePerGB;
+      remaining -= gbInTier;
+
+      if (remaining <= 0) {
+        dataTransfer.currentTier = i;
+        break;
+      }
+    }
+
+    return cost;
+  },
+
+  updateTotal () {
+    this.costs.total =
+      this.costs.requests.subtotal +
+      this.costs.storage.subtotal +
+      this.costs.dataTransfer.subtotal;
+
+    // Mirror to client.costs
+    if (this.client && this.client.costs) {
+      this.client.costs.total =
+        this.client.costs.requests.subtotal +
+        this.client.costs.storage.subtotal +
+        this.client.costs.dataTransfer.subtotal;
     }
   },
 }

package/src/plugins/eventual-consistency/helpers.js

@@ -9,7 +9,7 @@ import { getCohortInfo, resolveFieldAndPlugin } from "./utils.js";
 
 /**
  * Add helper methods to resources
- * This adds: set(), add(), sub(), consolidate(), getConsolidatedValue(), recalculate()
+ * This adds: set(), add(), sub(), increment(), decrement(), consolidate(), getConsolidatedValue(), recalculate()
  *
  * @param {Object} resource - Resource to add methods to
  * @param {Object} plugin - Plugin instance
@@ -137,6 +137,20 @@ export function addHelperMethods(resource, plugin, config) {
     return currentValue - amount;
   };
 
+  // Add method to increment value by 1 (shorthand for add(id, field, 1))
+  // Signature: increment(id, field)
+  // Supports dot notation: increment(id, 'loginCount')
+  resource.increment = async (id, field) => {
+    return await resource.add(id, field, 1);
+  };
+
+  // Add method to decrement value by 1 (shorthand for sub(id, field, 1))
+  // Signature: decrement(id, field)
+  // Supports dot notation: decrement(id, 'remainingAttempts')
+  resource.decrement = async (id, field) => {
+    return await resource.sub(id, field, 1);
+  };
+
   // Add method to manually trigger consolidation
   // Signature: consolidate(id, field)
   resource.consolidate = async (id, field) => {