s3db.js 18.0.11-next.1534f717 → 18.0.11-next.e8e71b5b

package/src/database/database-connection.class.ts

@@ -1,4 +1,4 @@
-import { PromisePool } from '@supercharge/promise-pool';
+import { TasksPool } from '../tasks/tasks-pool.class.js';
 import Resource from '../resource.class.js';
 import tryFn, { tryFnSync } from '../concerns/try-fn.js';
 import { streamToString } from '../stream/index.js';
@@ -182,16 +182,17 @@ export class DatabaseConnection {
         }
 
         const stopConcurrency = Math.max(1, Number.isFinite(db.executorPool?.concurrency) ? db.executorPool.concurrency! : 5);
-        await PromisePool
-          .withConcurrency(stopConcurrency)
-          .for(db.pluginList)
-          .process(async (plugin) => {
+        await TasksPool.map(
+          db.pluginList,
+          async (plugin) => {
             await tryFn(async () => {
               if (plugin && typeof (plugin as any).stop === 'function') {
                 await (plugin as any).stop();
               }
             });
-          });
+          },
+          { concurrency: stopConcurrency }
+        );
       }
 
       if (db.resources && Object.keys(db.resources).length > 0) {

package/src/database/database-plugins.class.ts

@@ -1,5 +1,5 @@
 import { isEmpty, isFunction } from 'lodash-es';
-import { PromisePool } from '@supercharge/promise-pool';
+import { TasksPool } from '../tasks/tasks-pool.class.js';
 import { DatabaseError } from '../errors.js';
 import type { DatabaseRef, Plugin, PluginConstructor, MemorySnapshot } from './types.js';
 import type { DatabaseCoordinators } from './database-coordinators.class.js';
@@ -31,10 +31,9 @@ export class DatabasePlugins {
 
       const concurrency = Math.max(1, Number.isFinite(db.executorPool?.concurrency) ? db.executorPool.concurrency! : 5);
 
-      const installResult = await PromisePool
-        .withConcurrency(concurrency)
-        .for(plugins)
-        .process(async (plugin) => {
+      const installResult = await TasksPool.map(
+        plugins,
+        async (plugin) => {
           const pluginName = this._getPluginName(plugin);
 
           if (typeof plugin.setInstanceName === 'function') {
@@ -53,12 +52,14 @@ export class DatabasePlugins {
 
           db.pluginRegistry[pluginName] = plugin;
           return pluginName;
-        });
+        },
+        { concurrency }
+      );
 
       if (installResult.errors.length > 0) {
         const errorInfo = installResult.errors[0]!;
         const failedPlugin = errorInfo.item;
-        const error = (errorInfo as any).raw || (errorInfo as any).error || errorInfo;
+        const error = errorInfo.error;
         const failedName = this._getPluginName(failedPlugin);
         throw new DatabaseError(`Failed to install plugin '${failedName}': ${error?.message || error}`, {
           operation: 'startPlugins.install',
@@ -67,10 +68,9 @@ export class DatabasePlugins {
         });
       }
 
-      const startResult = await PromisePool
-        .withConcurrency(concurrency)
-        .for(plugins)
-        .process(async (plugin) => {
+      const startResult = await TasksPool.map(
+        plugins,
+        async (plugin) => {
           const pluginName = this._getPluginName(plugin);
           await plugin.start();
           db.emit('db:plugin:metrics', {
@@ -79,12 +79,14 @@ export class DatabasePlugins {
             ...this.coordinators.collectMemorySnapshot()
           });
           return plugin;
-        });
+        },
+        { concurrency }
+      );
 
       if (startResult.errors.length > 0) {
         const errorInfo = startResult.errors[0]!;
         const failedPlugin = errorInfo.item;
-        const error = (errorInfo as any).raw || (errorInfo as any).error || errorInfo;
+        const error = errorInfo.error;
         const failedName = this._getPluginName(failedPlugin);
         throw new DatabaseError(`Failed to start plugin '${failedName}': ${error?.message || error}`, {
           operation: 'startPlugins.start',

package/src/plugins/concerns/s3-mutex.class.ts

@@ -0,0 +1,228 @@
+import { PluginStorage } from '../../concerns/plugin-storage.js';
+
+export interface LockResult {
+  acquired: boolean;
+  lockId?: string;
+  expiresAt?: number;
+  error?: Error;
+}
+
+export interface LockData {
+  lockId: string;
+  holderId: string;
+  acquiredAt: number;
+  expiresAt: number;
+}
+
+const DEFAULT_TTL_MS = 30000;
+const DEFAULT_NAMESPACE = 'default';
+
+export class S3Mutex {
+  protected storage: PluginStorage;
+  protected namespace: string;
+  protected holderId: string;
+
+  constructor(storage: PluginStorage, namespace?: string) {
+    if (!storage) {
+      throw new Error('S3Mutex: storage is required');
+    }
+    this.storage = storage;
+    this.namespace = namespace || DEFAULT_NAMESPACE;
+    this.holderId = this._generateHolderId();
+  }
+
+  async lock(key: string, ttlMs: number = DEFAULT_TTL_MS): Promise<LockResult> {
+    return this.tryLock(key, ttlMs);
+  }
+
+  async tryLock(key: string, ttlMs: number = DEFAULT_TTL_MS): Promise<LockResult> {
+    if (!key) {
+      return {
+        acquired: false,
+        error: new Error('S3Mutex: key is required')
+      };
+    }
+
+    const lockKey = this._getLockKey(key);
+    const now = Date.now();
+    const expiresAt = now + ttlMs;
+    const lockId = this._generateLockId();
+
+    const lockData: LockData = {
+      lockId,
+      holderId: this.holderId,
+      acquiredAt: now,
+      expiresAt
+    };
+
+    const version = await this.storage.setIfNotExists(
+      lockKey,
+      lockData as unknown as Record<string, unknown>,
+      { ttl: Math.ceil(ttlMs / 1000) + 60, behavior: 'body-only' }
+    );
+
+    if (version !== null) {
+      return {
+        acquired: true,
+        lockId,
+        expiresAt
+      };
+    }
+
+    const existingResult = await this.storage.getWithVersion(lockKey);
+
+    if (!existingResult.data) {
+      return {
+        acquired: false,
+        error: new Error('Lock exists but could not be read')
+      };
+    }
+
+    const existingLock = existingResult.data as unknown as LockData;
+
+    if (existingLock.expiresAt <= now) {
+      const newVersion = await this.storage.setIfVersion(
+        lockKey,
+        lockData as unknown as Record<string, unknown>,
+        existingResult.version!,
+        { ttl: Math.ceil(ttlMs / 1000) + 60, behavior: 'body-only' }
+      );
+
+      if (newVersion !== null) {
+        return {
+          acquired: true,
+          lockId,
+          expiresAt
+        };
+      }
+
+      return {
+        acquired: false,
+        error: new Error('Lock was taken by another process during expired lock takeover')
+      };
+    }
+
+    return {
+      acquired: false,
+      error: new Error(`Lock is held by ${existingLock.holderId} until ${new Date(existingLock.expiresAt).toISOString()}`)
+    };
+  }
+
+  async unlock(key: string, lockId: string): Promise<boolean> {
+    if (!key || !lockId) {
+      return false;
+    }
+
+    const lockKey = this._getLockKey(key);
+
+    const result = await this.storage.getWithVersion(lockKey);
+
+    if (!result.data || !result.version) {
+      return false;
+    }
+
+    const existingLock = result.data as unknown as LockData;
+
+    if (existingLock.lockId !== lockId) {
+      return false;
+    }
+
+    return await this.storage.deleteIfVersion(lockKey, result.version);
+  }
+
+  async isLocked(key: string): Promise<boolean> {
+    if (!key) {
+      return false;
+    }
+
+    const lockKey = this._getLockKey(key);
+    const result = await this.storage.getWithVersion(lockKey);
+
+    if (!result.data) {
+      return false;
+    }
+
+    const lockData = result.data as unknown as LockData;
+    const now = Date.now();
+
+    return lockData.expiresAt > now;
+  }
+
+  async extend(key: string, lockId: string, ttlMs: number): Promise<boolean> {
+    if (!key || !lockId || ttlMs <= 0) {
+      return false;
+    }
+
+    const lockKey = this._getLockKey(key);
+    const result = await this.storage.getWithVersion(lockKey);
+
+    if (!result.data || !result.version) {
+      return false;
+    }
+
+    const existingLock = result.data as unknown as LockData;
+    const now = Date.now();
+
+    if (existingLock.lockId !== lockId) {
+      return false;
+    }
+
+    if (existingLock.expiresAt <= now) {
+      return false;
+    }
+
+    const newExpiresAt = now + ttlMs;
+    const updatedLock: LockData = {
+      ...existingLock,
+      expiresAt: newExpiresAt
+    };
+
+    const newVersion = await this.storage.setIfVersion(
+      lockKey,
+      updatedLock as unknown as Record<string, unknown>,
+      result.version,
+      { ttl: Math.ceil(ttlMs / 1000) + 60, behavior: 'body-only' }
+    );
+
+    return newVersion !== null;
+  }
+
+  async getLockInfo(key: string): Promise<LockData | null> {
+    if (!key) {
+      return null;
+    }
+
+    const lockKey = this._getLockKey(key);
+    const result = await this.storage.getWithVersion(lockKey);
+
+    if (!result.data) {
+      return null;
+    }
+
+    return result.data as unknown as LockData;
+  }
+
+  protected _getLockKey(key: string): string {
+    return this.storage.getPluginKey(null, 'locks', `namespace=${this.namespace}`, `${key}.json`);
+  }
+
+  protected _generateLockId(): string {
+    const timestamp = Date.now();
+    const random = Math.random().toString(36).substring(2, 10);
+    return `lock-${this.holderId}-${timestamp}-${random}`;
+  }
+
+  protected _generateHolderId(): string {
+    if (typeof process !== 'undefined' && process.env) {
+      if (process.env.POD_NAME) {
+        return `holder-${process.env.POD_NAME}`;
+      }
+      if (process.env.HOSTNAME) {
+        return `holder-${process.env.HOSTNAME}`;
+      }
+    }
+    return `holder-${Date.now()}-${Math.random().toString(36).substring(2, 8)}`;
+  }
+}
+
+export default S3Mutex;

package/src/plugins/eventual-consistency/consolidation.ts

@@ -4,7 +4,7 @@
  */
 
 import tryFn from '../../concerns/try-fn.js';
-import { PromisePool } from '@supercharge/promise-pool';
+import { TasksPool } from '../../tasks/tasks-pool.class.js';
 import { getCronManager } from '../../concerns/cron-manager.js';
 import { createLogger } from '../../concerns/logger.js';
 import { PluginError } from '../../errors.js';
@@ -141,17 +141,18 @@ export async function runConsolidation(
 
     const byOriginalId = groupByOriginalId(allTransactions);
 
-    const { results, errors } = await PromisePool
-      .for(Object.entries(byOriginalId))
-      .withConcurrency(10)
-      .process(async ([originalId, transactions]) => {
+    const { results, errors } = await TasksPool.map(
+      Object.entries(byOriginalId),
+      async ([originalId, transactions]) => {
         return consolidateRecord(
           handler,
           originalId,
           transactions,
           config
         );
-      });
+      },
+      { concurrency: 10 }
+    );
 
     for (const recordResult of results) {
       if (recordResult) {
@@ -160,7 +161,7 @@ export async function runConsolidation(
       }
     }
 
-    result.errors = errors;
+    result.errors = errors.map(e => e.error);
     result.success = errors.length === 0;
 
     if (config.enableAnalytics && handler.analyticsResource) {

package/src/plugins/eventual-consistency/garbage-collection.ts

@@ -4,7 +4,7 @@
  */
 
 import tryFn from '../../concerns/try-fn.js';
-import { PromisePool } from '@supercharge/promise-pool';
+import { TasksPool } from '../../tasks/tasks-pool.class.js';
 import { getCronManager } from '../../concerns/cron-manager.js';
 import type { FieldHandler, Transaction } from './utils.js';
 import type { NormalizedConfig } from './config.js';
@@ -105,13 +105,14 @@ export async function runGarbageCollection(
       return;
     }
 
-    const { results, errors } = await PromisePool
-      .for(oldTransactions)
-      .withConcurrency(10)
-      .process(async (txn) => {
+    const { results, errors } = await TasksPool.map(
+      oldTransactions,
+      async (txn) => {
         const [deleted] = await tryFn(() => transactionResource!.delete(txn.id));
         return deleted;
-      });
+      },
+      { concurrency: 10 }
+    );
 
     if (emitFn) {
       emitFn('plg:eventual-consistency:gc-completed', {

package/src/plugins/queue-consumer.plugin.ts

@@ -1,4 +1,4 @@
-import { PromisePool } from '@supercharge/promise-pool';
+import { TasksPool } from '../tasks/tasks-pool.class.js';
 import { Plugin } from './plugin.class.js';
 import { createConsumer } from './consumers/index.js';
 import tryFn from '../concerns/try-fn.js';
@@ -124,20 +124,21 @@ export class QueueConsumerPlugin extends Plugin {
       return;
     }
 
-    const { errors } = await PromisePool
-      .withConcurrency(this.startConcurrency)
-      .for(startTasks)
-      .process(async (task: StartTask) => {
+    const { errors } = await TasksPool.map(
+      startTasks,
+      async (task: StartTask) => {
         await task.start();
         return `${task.driver}:${task.resource}`;
-      });
+      },
+      { concurrency: this.startConcurrency }
+    );
 
     if (errors.length > 0) {
-      const messages = errors.map((error) => {
-        const task = error.item as StartTask | undefined;
-        const reason = (error as any).reason ?? error;
+      const messages = errors.map((errorInfo) => {
+        const task = errorInfo.item as StartTask | undefined;
+        const reason = errorInfo.error;
         const identifier = task ? `${task.driver || 'unknown'}:${task.resource || 'unknown'}` : 'unknown';
-        return `[${identifier}] ${(reason as Error)?.message || reason}`;
+        return `[${identifier}] ${reason?.message || reason}`;
       });
 
       throw new QueueError('Failed to start one or more queue consumers', {
@@ -161,20 +162,21 @@ export class QueueConsumerPlugin extends Plugin {
         stop: () => consumer.stop()
       }));
 
-    const { errors } = await PromisePool
-      .withConcurrency(this.stopConcurrency)
-      .for(stopTasks)
-      .process(async (task: StopTask) => {
+    const { errors } = await TasksPool.map(
+      stopTasks,
+      async (task: StopTask) => {
         await task.stop();
         return task.consumer;
-      });
+      },
+      { concurrency: this.stopConcurrency }
+    );
 
     if (errors.length > 0) {
-      errors.forEach((error) => {
-        const reason = (error as any).reason ?? error;
+      errors.forEach((errorInfo) => {
+        const reason = errorInfo.error;
         this.logger.warn(
-          { error: (reason as Error)?.message || reason },
-          `Failed to stop consumer: ${(reason as Error)?.message || reason}`
+          { error: reason?.message || reason },
+          `Failed to stop consumer: ${reason?.message || reason}`
         );
       });
     }

package/src/plugins/recon/managers/scheduler-manager.ts

@@ -7,7 +7,7 @@
  * - Iterates over enabled targets
  */
 
-import { PromisePool } from '@supercharge/promise-pool';
+import { TasksPool } from '../../../tasks/tasks-pool.class.js';
 import { getCronManager } from '../../../concerns/cron-manager.js';
 import type { TargetManager, TargetRecord } from './target-manager.js';
 
@@ -149,9 +149,9 @@ export class SchedulerManager {
       targets: activeTargets.map(t => t.id)
     });
 
-    await PromisePool.withConcurrency(this.plugin.config.concurrency || 1)
-      .for(activeTargets)
-      .process(async (targetEntry) => {
+    await TasksPool.map(
+      activeTargets,
+      async (targetEntry) => {
         try {
           const report = await this.plugin.runDiagnostics(targetEntry.target, {
             behavior: targetEntry.behavior,
@@ -177,7 +177,9 @@ export class SchedulerManager {
             error
           });
         }
-      });
+      },
+      { concurrency: this.plugin.config.concurrency || 1 }
+    );
 
     this.plugin.emit('recon:sweep-completed', {
       reason,