s3db.js 9.2.0 → 9.2.2

package/src/plugins/backup/multi-backup-driver.class.js

@@ -0,0 +1,304 @@
+import BaseBackupDriver from './base-backup-driver.class.js';
+import { createBackupDriver } from './index.js';
+import tryFn from '../../concerns/try-fn.js';
+
+/**
+ * MultiBackupDriver - Manages multiple backup destinations
+ *
+ * Configuration:
+ * - destinations: Array of driver configurations
+ *   - driver: Driver type (filesystem, s3)
+ *   - config: Driver-specific configuration
+ * - strategy: Backup strategy (default: 'all')
+ *   - 'all': Upload to all destinations (fail if any fails)
+ *   - 'any': Upload to all, succeed if at least one succeeds
+ *   - 'priority': Try destinations in order, stop on first success
+ * - concurrency: Max concurrent uploads (default: 3)
+ */
+export default class MultiBackupDriver extends BaseBackupDriver {
+  constructor(config = {}) {
+    super({
+      destinations: [],
+      strategy: 'all', // 'all', 'any', 'priority'
+      concurrency: 3,
+      requireAll: true, // For backward compatibility
+      ...config
+    });
+
+    this.drivers = [];
+  }
+
+  getType() {
+    return 'multi';
+  }
+
+  async onSetup() {
+    if (!Array.isArray(this.config.destinations) || this.config.destinations.length === 0) {
+      throw new Error('MultiBackupDriver: destinations array is required and must not be empty');
+    }
+
+    // Create and setup all driver instances
+    for (const [index, destConfig] of this.config.destinations.entries()) {
+      if (!destConfig.driver) {
+        throw new Error(`MultiBackupDriver: destination[${index}] must have a driver type`);
+      }
+
+      try {
+        const driver = createBackupDriver(destConfig.driver, destConfig.config || {});
+        await driver.setup(this.database);
+        this.drivers.push({
+          driver,
+          config: destConfig,
+          index
+        });
+        
+        this.log(`Setup destination ${index}: ${destConfig.driver}`);
+      } catch (error) {
+        throw new Error(`Failed to setup destination ${index} (${destConfig.driver}): ${error.message}`);
+      }
+    }
+
+    // Legacy support for requireAll
+    if (this.config.requireAll === false) {
+      this.config.strategy = 'any';
+    }
+
+    this.log(`Initialized with ${this.drivers.length} destinations, strategy: ${this.config.strategy}`);
+  }
+
+  async upload(filePath, backupId, manifest) {
+    const strategy = this.config.strategy;
+    const results = [];
+    const errors = [];
+
+    if (strategy === 'priority') {
+      // Try destinations in order, stop on first success
+      for (const { driver, config, index } of this.drivers) {
+        const [ok, err, result] = await tryFn(() => 
+          driver.upload(filePath, backupId, manifest)
+        );
+
+        if (ok) {
+          this.log(`Priority upload successful to destination ${index}`);
+          return [{
+            ...result,
+            driver: config.driver,
+            destination: index,
+            status: 'success'
+          }];
+        } else {
+          errors.push({ destination: index, error: err.message });
+          this.log(`Priority upload failed to destination ${index}: ${err.message}`);
+        }
+      }
+
+      throw new Error(`All priority destinations failed: ${errors.map(e => `${e.destination}: ${e.error}`).join('; ')}`);
+    }
+
+    // For 'all' and 'any' strategies, upload to all destinations
+    const uploadPromises = this.drivers.map(async ({ driver, config, index }) => {
+      const [ok, err, result] = await tryFn(() => 
+        driver.upload(filePath, backupId, manifest)
+      );
+
+      if (ok) {
+        this.log(`Upload successful to destination ${index}`);
+        return {
+          ...result,
+          driver: config.driver,
+          destination: index,
+          status: 'success'
+        };
+      } else {
+        this.log(`Upload failed to destination ${index}: ${err.message}`);
+        const errorResult = {
+          driver: config.driver,
+          destination: index,
+          status: 'failed',
+          error: err.message
+        };
+        errors.push(errorResult);
+        return errorResult;
+      }
+    });
+
+    // Execute uploads with concurrency limit
+    const allResults = await this._executeConcurrent(uploadPromises, this.config.concurrency);
+    const successResults = allResults.filter(r => r.status === 'success');
+    const failedResults = allResults.filter(r => r.status === 'failed');
+
+    if (strategy === 'all' && failedResults.length > 0) {
+      throw new Error(`Some destinations failed: ${failedResults.map(r => `${r.destination}: ${r.error}`).join('; ')}`);
+    }
+
+    if (strategy === 'any' && successResults.length === 0) {
+      throw new Error(`All destinations failed: ${failedResults.map(r => `${r.destination}: ${r.error}`).join('; ')}`);
+    }
+
+    return allResults;
+  }
+
+  async download(backupId, targetPath, metadata) {
+    // Try to download from the first available destination
+    const destinations = Array.isArray(metadata.destinations) ? metadata.destinations : [metadata];
+
+    for (const destMetadata of destinations) {
+      if (destMetadata.status !== 'success') continue;
+
+      const driverInstance = this.drivers.find(d => d.index === destMetadata.destination);
+      if (!driverInstance) continue;
+
+      const [ok, err, result] = await tryFn(() => 
+        driverInstance.driver.download(backupId, targetPath, destMetadata)
+      );
+
+      if (ok) {
+        this.log(`Downloaded from destination ${destMetadata.destination}`);
+        return result;
+      } else {
+        this.log(`Download failed from destination ${destMetadata.destination}: ${err.message}`);
+      }
+    }
+
+    throw new Error(`Failed to download backup from any destination`);
+  }
+
+  async delete(backupId, metadata) {
+    const destinations = Array.isArray(metadata.destinations) ? metadata.destinations : [metadata];
+    const errors = [];
+    let successCount = 0;
+
+    for (const destMetadata of destinations) {
+      if (destMetadata.status !== 'success') continue;
+
+      const driverInstance = this.drivers.find(d => d.index === destMetadata.destination);
+      if (!driverInstance) continue;
+
+      const [ok, err] = await tryFn(() => 
+        driverInstance.driver.delete(backupId, destMetadata)
+      );
+
+      if (ok) {
+        successCount++;
+        this.log(`Deleted from destination ${destMetadata.destination}`);
+      } else {
+        errors.push(`${destMetadata.destination}: ${err.message}`);
+        this.log(`Delete failed from destination ${destMetadata.destination}: ${err.message}`);
+      }
+    }
+
+    if (successCount === 0 && errors.length > 0) {
+      throw new Error(`Failed to delete from any destination: ${errors.join('; ')}`);
+    }
+
+    if (errors.length > 0) {
+      this.log(`Partial delete success, some errors: ${errors.join('; ')}`);
+    }
+  }
+
+  async list(options = {}) {
+    // Get lists from all destinations and merge/deduplicate
+    const allLists = await Promise.allSettled(
+      this.drivers.map(({ driver, index }) => 
+        driver.list(options).catch(err => {
+          this.log(`List failed for destination ${index}: ${err.message}`);
+          return [];
+        })
+      )
+    );
+
+    const backupMap = new Map();
+
+    // Merge results from all destinations
+    allLists.forEach((result, index) => {
+      if (result.status === 'fulfilled') {
+        result.value.forEach(backup => {
+          const existing = backupMap.get(backup.id);
+          if (!existing || new Date(backup.createdAt) > new Date(existing.createdAt)) {
+            backupMap.set(backup.id, {
+              ...backup,
+              destinations: existing ? [...(existing.destinations || []), { destination: index, ...backup }] : [{ destination: index, ...backup }]
+            });
+          }
+        });
+      }
+    });
+
+    const results = Array.from(backupMap.values())
+      .sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt))
+      .slice(0, options.limit || 50);
+
+    return results;
+  }
+
+  async verify(backupId, expectedChecksum, metadata) {
+    const destinations = Array.isArray(metadata.destinations) ? metadata.destinations : [metadata];
+    
+    // Verify against any successful destination
+    for (const destMetadata of destinations) {
+      if (destMetadata.status !== 'success') continue;
+
+      const driverInstance = this.drivers.find(d => d.index === destMetadata.destination);
+      if (!driverInstance) continue;
+
+      const [ok, , isValid] = await tryFn(() => 
+        driverInstance.driver.verify(backupId, expectedChecksum, destMetadata)
+      );
+
+      if (ok && isValid) {
+        this.log(`Verification successful from destination ${destMetadata.destination}`);
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  async cleanup() {
+    await Promise.all(
+      this.drivers.map(({ driver }) => 
+        tryFn(() => driver.cleanup()).catch(() => {})
+      )
+    );
+  }
+
+  getStorageInfo() {
+    return {
+      ...super.getStorageInfo(),
+      strategy: this.config.strategy,
+      destinations: this.drivers.map(({ driver, config, index }) => ({
+        index,
+        driver: config.driver,
+        info: driver.getStorageInfo()
+      }))
+    };
+  }
+
+  /**
+   * Execute promises with concurrency limit
+   * @param {Array} promises - Array of promise functions
+   * @param {number} concurrency - Max concurrent executions
+   * @returns {Array} Results in original order
+   */
+  async _executeConcurrent(promises, concurrency) {
+    const results = new Array(promises.length);
+    const executing = [];
+
+    for (let i = 0; i < promises.length; i++) {
+      const promise = Promise.resolve(promises[i]).then(result => {
+        results[i] = result;
+        return result;
+      });
+
+      executing.push(promise);
+
+      if (executing.length >= concurrency) {
+        await Promise.race(executing);
+        executing.splice(executing.findIndex(p => p === promise), 1);
+      }
+    }
+
+    await Promise.all(executing);
+    return results;
+  }
+}

package/src/plugins/backup/s3-backup-driver.class.js

@@ -0,0 +1,313 @@
+import BaseBackupDriver from './base-backup-driver.class.js';
+import { createReadStream } from 'fs';
+import { stat } from 'fs/promises';
+import path from 'path';
+import crypto from 'crypto';
+import tryFn from '../../concerns/try-fn.js';
+
+/**
+ * S3BackupDriver - Stores backups in S3-compatible storage
+ *
+ * Configuration:
+ * - bucket: S3 bucket name (optional, uses database bucket if not specified)
+ * - path: Key prefix for backups (supports template variables)
+ * - storageClass: S3 storage class (default: STANDARD_IA)
+ * - serverSideEncryption: S3 server-side encryption (default: AES256)
+ * - client: Custom S3 client (optional, uses database client if not specified)
+ */
+export default class S3BackupDriver extends BaseBackupDriver {
+  constructor(config = {}) {
+    super({
+      bucket: null, // Will use database bucket if not specified
+      path: 'backups/{date}/',
+      storageClass: 'STANDARD_IA',
+      serverSideEncryption: 'AES256',
+      client: null, // Will use database client if not specified
+      ...config
+    });
+  }
+
+  getType() {
+    return 's3';
+  }
+
+  async onSetup() {
+    // Use database client if not provided
+    if (!this.config.client) {
+      this.config.client = this.database.client;
+    }
+
+    // Use database bucket if not specified
+    if (!this.config.bucket) {
+      this.config.bucket = this.database.bucket;
+    }
+
+    if (!this.config.client) {
+      throw new Error('S3BackupDriver: client is required (either via config or database)');
+    }
+
+    if (!this.config.bucket) {
+      throw new Error('S3BackupDriver: bucket is required (either via config or database)');
+    }
+
+    this.log(`Initialized with bucket: ${this.config.bucket}, path: ${this.config.path}`);
+  }
+
+  /**
+   * Resolve S3 key template variables
+   * @param {string} backupId - Backup identifier
+   * @param {Object} manifest - Backup manifest
+   * @returns {string} Resolved S3 key
+   */
+  resolveKey(backupId, manifest = {}) {
+    const now = new Date();
+    const dateStr = now.toISOString().slice(0, 10); // YYYY-MM-DD
+    const timeStr = now.toISOString().slice(11, 19).replace(/:/g, '-'); // HH-MM-SS
+    
+    const basePath = this.config.path
+      .replace('{date}', dateStr)
+      .replace('{time}', timeStr)
+      .replace('{year}', now.getFullYear().toString())
+      .replace('{month}', (now.getMonth() + 1).toString().padStart(2, '0'))
+      .replace('{day}', now.getDate().toString().padStart(2, '0'))
+      .replace('{backupId}', backupId)
+      .replace('{type}', manifest.type || 'backup');
+
+    return path.posix.join(basePath, `${backupId}.backup`);
+  }
+
+  resolveManifestKey(backupId, manifest = {}) {
+    return this.resolveKey(backupId, manifest).replace('.backup', '.manifest.json');
+  }
+
+  async upload(filePath, backupId, manifest) {
+    const backupKey = this.resolveKey(backupId, manifest);
+    const manifestKey = this.resolveManifestKey(backupId, manifest);
+
+    // Get file size
+    const [statOk, , stats] = await tryFn(() => stat(filePath));
+    const fileSize = statOk ? stats.size : 0;
+
+    // Upload backup file
+    const [uploadOk, uploadErr] = await tryFn(async () => {
+      const fileStream = createReadStream(filePath);
+      
+      return await this.config.client.uploadObject({
+        bucket: this.config.bucket,
+        key: backupKey,
+        body: fileStream,
+        contentLength: fileSize,
+        metadata: {
+          'backup-id': backupId,
+          'backup-type': manifest.type || 'backup',
+          'created-at': new Date().toISOString()
+        },
+        storageClass: this.config.storageClass,
+        serverSideEncryption: this.config.serverSideEncryption
+      });
+    });
+
+    if (!uploadOk) {
+      throw new Error(`Failed to upload backup file: ${uploadErr.message}`);
+    }
+
+    // Upload manifest
+    const [manifestOk, manifestErr] = await tryFn(() => 
+      this.config.client.uploadObject({
+        bucket: this.config.bucket,
+        key: manifestKey,
+        body: JSON.stringify(manifest, null, 2),
+        contentType: 'application/json',
+        metadata: {
+          'backup-id': backupId,
+          'manifest-for': backupKey
+        },
+        storageClass: this.config.storageClass,
+        serverSideEncryption: this.config.serverSideEncryption
+      })
+    );
+
+    if (!manifestOk) {
+      // Clean up backup file if manifest upload fails
+      await tryFn(() => this.config.client.deleteObject({
+        bucket: this.config.bucket,
+        key: backupKey
+      }));
+      throw new Error(`Failed to upload manifest: ${manifestErr.message}`);
+    }
+
+    this.log(`Uploaded backup ${backupId} to s3://${this.config.bucket}/${backupKey} (${fileSize} bytes)`);
+
+    return {
+      bucket: this.config.bucket,
+      key: backupKey,
+      manifestKey,
+      size: fileSize,
+      storageClass: this.config.storageClass,
+      uploadedAt: new Date().toISOString(),
+      etag: uploadOk?.ETag
+    };
+  }
+
+  async download(backupId, targetPath, metadata) {
+    const backupKey = metadata.key || this.resolveKey(backupId, metadata);
+
+    const [downloadOk, downloadErr] = await tryFn(() => 
+      this.config.client.downloadObject({
+        bucket: this.config.bucket,
+        key: backupKey,
+        filePath: targetPath
+      })
+    );
+
+    if (!downloadOk) {
+      throw new Error(`Failed to download backup: ${downloadErr.message}`);
+    }
+
+    this.log(`Downloaded backup ${backupId} from s3://${this.config.bucket}/${backupKey} to ${targetPath}`);
+    return targetPath;
+  }
+
+  async delete(backupId, metadata) {
+    const backupKey = metadata.key || this.resolveKey(backupId, metadata);
+    const manifestKey = metadata.manifestKey || this.resolveManifestKey(backupId, metadata);
+
+    // Delete backup file
+    const [deleteBackupOk] = await tryFn(() => 
+      this.config.client.deleteObject({
+        bucket: this.config.bucket,
+        key: backupKey
+      })
+    );
+
+    // Delete manifest
+    const [deleteManifestOk] = await tryFn(() => 
+      this.config.client.deleteObject({
+        bucket: this.config.bucket,
+        key: manifestKey
+      })
+    );
+
+    if (!deleteBackupOk && !deleteManifestOk) {
+      throw new Error(`Failed to delete backup objects for ${backupId}`);
+    }
+
+    this.log(`Deleted backup ${backupId} from S3`);
+  }
+
+  async list(options = {}) {
+    const { limit = 50, prefix = '' } = options;
+    const searchPrefix = this.config.path.replace(/\{[^}]+\}/g, '');
+    
+    const [listOk, listErr, response] = await tryFn(() => 
+      this.config.client.listObjects({
+        bucket: this.config.bucket,
+        prefix: searchPrefix,
+        maxKeys: limit * 2 // Get more to account for manifest files
+      })
+    );
+
+    if (!listOk) {
+      this.log(`Error listing S3 objects: ${listErr.message}`);
+      return [];
+    }
+
+    const manifestObjects = (response.Contents || [])
+      .filter(obj => obj.Key.endsWith('.manifest.json'))
+      .filter(obj => !prefix || obj.Key.includes(prefix));
+
+    const results = [];
+    
+    for (const obj of manifestObjects.slice(0, limit)) {
+      const [manifestOk, , manifestContent] = await tryFn(() => 
+        this.config.client.getObject({
+          bucket: this.config.bucket,
+          key: obj.Key
+        })
+      );
+
+      if (manifestOk) {
+        try {
+          const manifest = JSON.parse(manifestContent);
+          const backupId = path.basename(obj.Key, '.manifest.json');
+          
+          results.push({
+            id: backupId,
+            bucket: this.config.bucket,
+            key: obj.Key.replace('.manifest.json', '.backup'),
+            manifestKey: obj.Key,
+            size: obj.Size,
+            lastModified: obj.LastModified,
+            storageClass: obj.StorageClass,
+            createdAt: manifest.createdAt || obj.LastModified,
+            ...manifest
+          });
+        } catch (parseErr) {
+          this.log(`Failed to parse manifest ${obj.Key}: ${parseErr.message}`);
+        }
+      }
+    }
+
+    // Sort by creation time (newest first)
+    results.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt));
+    
+    return results;
+  }
+
+  async verify(backupId, expectedChecksum, metadata) {
+    const backupKey = metadata.key || this.resolveKey(backupId, metadata);
+
+    const [verifyOk, verifyErr] = await tryFn(async () => {
+      // Get object metadata to check ETag
+      const headResponse = await this.config.client.headObject({
+        bucket: this.config.bucket,
+        key: backupKey
+      });
+
+      // For single-part uploads, ETag is the MD5 hash
+      // For multipart uploads, ETag has a suffix like "-2"
+      const etag = headResponse.ETag?.replace(/"/g, '');
+      
+      if (etag && !etag.includes('-')) {
+        // Single-part upload, ETag is MD5
+        const expectedMd5 = crypto.createHash('md5').update(expectedChecksum).digest('hex');
+        return etag === expectedMd5;
+      } else {
+        // For multipart uploads or SHA256 comparison, download and verify
+        const [streamOk, , stream] = await tryFn(() => 
+          this.config.client.getObjectStream({
+            bucket: this.config.bucket,
+            key: backupKey
+          })
+        );
+
+        if (!streamOk) return false;
+
+        const hash = crypto.createHash('sha256');
+        for await (const chunk of stream) {
+          hash.update(chunk);
+        }
+        
+        const actualChecksum = hash.digest('hex');
+        return actualChecksum === expectedChecksum;
+      }
+    });
+
+    if (!verifyOk) {
+      this.log(`Verification failed for ${backupId}: ${verifyErr?.message || 'checksum mismatch'}`);
+      return false;
+    }
+
+    return true;
+  }
+
+  getStorageInfo() {
+    return {
+      ...super.getStorageInfo(),
+      bucket: this.config.bucket,
+      path: this.config.path,
+      storageClass: this.config.storageClass,
+      serverSideEncryption: this.config.serverSideEncryption
+    };
+  }
+}