s3db.js 10.0.5 → 10.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "10.0.5",
+  "version": "10.0.7",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",
@@ -79,16 +79,16 @@
     "validate:types": "pnpm run test:ts && echo 'TypeScript definitions are valid!'"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.873.0",
-    "@modelcontextprotocol/sdk": "^1.17.4",
-    "@smithy/node-http-handler": "^4.1.1",
+    "@aws-sdk/client-s3": "^3.906.0",
+    "@modelcontextprotocol/sdk": "^1.19.1",
+    "@smithy/node-http-handler": "^4.3.0",
     "@supercharge/promise-pool": "^3.2.0",
-    "dotenv": "^17.2.1",
+    "dotenv": "^17.2.3",
     "fastest-validator": "^1.19.1",
     "flat": "^6.0.1",
     "json-stable-stringify": "^1.3.0",
     "lodash-es": "^4.17.21",
-    "nanoid": "5.1.5"
+    "nanoid": "5.1.6"
   },
   "peerDependencies": {
     "@aws-sdk/client-sqs": "^3.0.0",
@@ -115,32 +115,32 @@
     }
   },
   "devDependencies": {
-    "@babel/core": "^7.28.3",
+    "@babel/core": "^7.28.4",
     "@babel/preset-env": "^7.28.3",
     "@rollup/plugin-commonjs": "^28.0.6",
     "@rollup/plugin-json": "^6.1.0",
-    "@rollup/plugin-node-resolve": "^16.0.1",
+    "@rollup/plugin-node-resolve": "^16.0.2",
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-terser": "^0.4.4",
-    "@types/node": "24.3.0",
+    "@types/node": "24.7.0",
     "babel-loader": "^10.0.0",
-    "chalk": "^5.6.0",
+    "chalk": "^5.6.2",
     "cli-table3": "^0.6.5",
-    "commander": "^14.0.0",
-    "esbuild": "^0.25.9",
-    "inquirer": "^12.9.3",
-    "jest": "^30.0.5",
+    "commander": "^14.0.1",
+    "esbuild": "^0.25.10",
+    "inquirer": "^12.9.6",
+    "jest": "^30.2.0",
     "node-loader": "^2.1.0",
-    "ora": "^8.2.0",
+    "ora": "^9.0.0",
     "pkg": "^5.8.1",
-    "rollup": "^4.48.0",
+    "rollup": "^4.52.4",
     "rollup-plugin-copy": "^3.5.0",
     "rollup-plugin-esbuild": "^6.2.1",
     "rollup-plugin-polyfill-node": "^0.13.0",
     "rollup-plugin-shebang-bin": "^0.1.0",
     "rollup-plugin-terser": "^7.0.2",
-    "typescript": "5.9.2",
-    "webpack": "^5.101.3",
+    "typescript": "5.9.3",
+    "webpack": "^5.102.1",
     "webpack-cli": "^6.0.1"
   },
   "funding": [

package/src/database.class.js

@@ -475,6 +475,7 @@ export class Database extends EventEmitter {
       metadata.resources[name] = {
         currentVersion: version,
         partitions: resource.config.partitions || {},
+        createdBy: existingResource?.createdBy || resource.config.createdBy || 'user',
         versions: {
           ...existingResource?.versions, // Preserve previous versions
           [version]: {
@@ -906,6 +907,7 @@ export class Database extends EventEmitter {
    * @param {boolean} [config.autoDecrypt=true] - Auto-decrypt secret fields
    * @param {Function|number} [config.idGenerator] - Custom ID generator or size
    * @param {number} [config.idSize=22] - Size for auto-generated IDs
+   * @param {string} [config.createdBy='user'] - Who created this resource ('user', 'plugin', or plugin name)
    * @returns {Promise<Resource>} The created or updated resource
    */
   async createResource({ name, attributes, behavior = 'user-managed', hooks, ...config }) {
@@ -968,7 +970,8 @@ export class Database extends EventEmitter {
       idGenerator: config.idGenerator,
       idSize: config.idSize,
       asyncEvents: config.asyncEvents,
-      events: config.events || {}
+      events: config.events || {},
+      createdBy: config.createdBy || 'user'
     });
     resource.database = this;
     this.resources[name] = resource;

package/src/plugins/cache.plugin.js

@@ -113,7 +113,15 @@ export class CachePlugin extends Plugin {
   }
 
   shouldCacheResource(resourceName) {
-    // Skip plugin resources by default (unless explicitly included)
+    // Get resource metadata to check createdBy
+    const resourceMetadata = this.database.savedMetadata?.resources?.[resourceName];
+
+    // Skip plugin-created resources by default (unless explicitly included)
+    if (resourceMetadata?.createdBy && resourceMetadata.createdBy !== 'user' && !this.config.include) {
+      return false;
+    }
+
+    // Legacy: Skip plugin resources by name pattern (unless explicitly included)
     if (resourceName.startsWith('plg_') && !this.config.include) {
       return false;
     }

package/src/plugins/eventual-consistency.plugin.js

@@ -144,7 +144,8 @@ export class EventualConsistencyPlugin extends Plugin {
         behavior: 'body-overflow',
         timestamps: true,
         partitions: partitionConfig,
-        asyncPartitions: true // Use async partitions for better performance
+        asyncPartitions: true, // Use async partitions for better performance
+        createdBy: 'EventualConsistencyPlugin'
       })
     );
 
@@ -165,7 +166,8 @@ export class EventualConsistencyPlugin extends Plugin {
           workerId: 'string|optional'
         },
         behavior: 'body-only',
-        timestamps: false
+        timestamps: false,
+        createdBy: 'EventualConsistencyPlugin'
       })
     );
 
@@ -186,10 +188,33 @@ export class EventualConsistencyPlugin extends Plugin {
     // Setup consolidation if enabled
     if (this.config.autoConsolidate) {
       this.startConsolidationTimer();
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Auto-consolidation ENABLED (interval: ${this.config.consolidationInterval}s, ` +
+          `window: ${this.config.consolidationWindow}h, mode: ${this.config.mode})`
+        );
+      }
+    } else {
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Auto-consolidation DISABLED (manual consolidation only)`
+        );
+      }
     }
 
     // Setup garbage collection timer
     this.startGarbageCollectionTimer();
+
+    if (this.config.verbose) {
+      console.log(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Setup complete. Resources: ${this.config.resource}_transactions_${this.config.field}, ` +
+        `${this.config.resource}_consolidation_locks_${this.config.field}` +
+        `${this.config.enableAnalytics ? `, ${this.config.resource}_analytics_${this.config.field}` : ''}`
+      );
+    }
   }
 
   async onStart() {
@@ -286,7 +311,8 @@ export class EventualConsistencyPlugin extends Plugin {
           byCohort: {
             fields: { cohort: 'string' }
           }
-        }
+        },
+        createdBy: 'EventualConsistencyPlugin'
       })
     );
 
@@ -539,15 +565,31 @@ export class EventualConsistencyPlugin extends Plugin {
     // Batch transactions if configured
     if (this.config.batchTransactions) {
       this.pendingTransactions.set(transaction.id, transaction);
-      
+
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Transaction batched: ${data.operation} ${data.value} for ${data.originalId} ` +
+          `(batch: ${this.pendingTransactions.size}/${this.config.batchSize})`
+        );
+      }
+
       // Flush if batch size reached
       if (this.pendingTransactions.size >= this.config.batchSize) {
         await this.flushPendingTransactions();
       }
     } else {
       await this.transactionResource.insert(transaction);
+
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Transaction created: ${data.operation} ${data.value} for ${data.originalId} ` +
+          `(cohort: ${cohortInfo.hour}, applied: false)`
+        );
+      }
     }
-    
+
     return transaction;
   }
 
@@ -636,12 +678,30 @@ export class EventualConsistencyPlugin extends Plugin {
   startConsolidationTimer() {
     const intervalMs = this.config.consolidationInterval * 1000; // Convert seconds to ms
 
+    if (this.config.verbose) {
+      const nextRun = new Date(Date.now() + intervalMs);
+      console.log(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Consolidation timer started. Next run at ${nextRun.toISOString()} ` +
+        `(every ${this.config.consolidationInterval}s)`
+      );
+    }
+
     this.consolidationTimer = setInterval(async () => {
       await this.runConsolidation();
     }, intervalMs);
   }
 
   async runConsolidation() {
+    const startTime = Date.now();
+
+    if (this.config.verbose) {
+      console.log(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Starting consolidation run at ${new Date().toISOString()}`
+      );
+    }
+
     try {
       // Query unapplied transactions from recent cohorts (last 24 hours by default)
       // This uses hourly partition for O(1) performance instead of full scan
@@ -655,6 +715,13 @@ export class EventualConsistencyPlugin extends Plugin {
         cohortHours.push(cohortInfo.hour);
       }
 
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Querying ${hoursToCheck} hour partitions for pending transactions...`
+        );
+      }
+
       // Query transactions by partition for each hour (parallel for speed)
       const transactionsByHour = await Promise.all(
         cohortHours.map(async (cohortHour) => {
@@ -673,7 +740,10 @@ export class EventualConsistencyPlugin extends Plugin {
 
       if (transactions.length === 0) {
         if (this.config.verbose) {
-          console.log(`[EventualConsistency] No pending transactions to consolidate`);
+          console.log(
+            `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+            `No pending transactions found. Next run in ${this.config.consolidationInterval}s`
+          );
         }
         return;
       }
@@ -681,6 +751,14 @@ export class EventualConsistencyPlugin extends Plugin {
       // Get unique originalIds
       const uniqueIds = [...new Set(transactions.map(t => t.originalId))];
 
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Found ${transactions.length} pending transactions for ${uniqueIds.length} records. ` +
+          `Consolidating with concurrency=${this.config.consolidationConcurrency}...`
+        );
+      }
+
       // Consolidate each record in parallel with concurrency limit
       const { results, errors } = await PromisePool
         .for(uniqueIds)
@@ -689,8 +767,22 @@ export class EventualConsistencyPlugin extends Plugin {
           return await this.consolidateRecord(id);
         });
 
+      const duration = Date.now() - startTime;
+
       if (errors && errors.length > 0) {
-        console.error(`Consolidation completed with ${errors.length} errors:`, errors);
+        console.error(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Consolidation completed with ${errors.length} errors in ${duration}ms:`,
+          errors
+        );
+      }
+
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Consolidation complete: ${results.length} records consolidated in ${duration}ms ` +
+          `(${errors.length} errors). Next run in ${this.config.consolidationInterval}s`
+        );
       }
 
       this.emit('eventual-consistency.consolidated', {
@@ -698,10 +790,16 @@ export class EventualConsistencyPlugin extends Plugin {
         field: this.config.field,
         recordCount: uniqueIds.length,
         successCount: results.length,
-        errorCount: errors.length
+        errorCount: errors.length,
+        duration
       });
     } catch (error) {
-      console.error('Consolidation error:', error);
+      const duration = Date.now() - startTime;
+      console.error(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Consolidation error after ${duration}ms:`,
+        error
+      );
       this.emit('eventual-consistency.consolidation-error', error);
     }
   }
@@ -749,9 +847,23 @@ export class EventualConsistencyPlugin extends Plugin {
       );
 
       if (!ok || !transactions || transactions.length === 0) {
+        if (this.config.verbose) {
+          console.log(
+            `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+            `No pending transactions for ${originalId}, skipping`
+          );
+        }
         return currentValue;
       }
 
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Consolidating ${originalId}: ${transactions.length} pending transactions ` +
+          `(current: ${currentValue})`
+        );
+      }
+
       // Sort transactions by timestamp
       transactions.sort((a, b) =>
         new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
@@ -766,6 +878,14 @@ export class EventualConsistencyPlugin extends Plugin {
       // Apply reducer to get consolidated value
       const consolidatedValue = this.config.reducer(transactions);
 
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `${originalId}: ${currentValue} → ${consolidatedValue} ` +
+          `(${consolidatedValue > currentValue ? '+' : ''}${consolidatedValue - currentValue})`
+        );
+      }
+
       // Update the original record
       const [updateOk, updateErr] = await tryFn(() =>
         this.targetResource.update(originalId, {
@@ -800,6 +920,29 @@ export class EventualConsistencyPlugin extends Plugin {
         if (this.config.enableAnalytics && transactionsToUpdate.length > 0) {
           await this.updateAnalytics(transactionsToUpdate);
         }
+
+        // Invalidate cache for this record after consolidation
+        if (this.targetResource.cache && typeof this.targetResource.cache.delete === 'function') {
+          try {
+            const cacheKey = await this.targetResource.cacheKeyFor({ id: originalId });
+            await this.targetResource.cache.delete(cacheKey);
+
+            if (this.config.verbose) {
+              console.log(
+                `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+                `Cache invalidated for ${originalId}`
+              );
+            }
+          } catch (cacheErr) {
+            // Log but don't fail consolidation if cache invalidation fails
+            if (this.config.verbose) {
+              console.warn(
+                `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+                `Failed to invalidate cache for ${originalId}: ${cacheErr?.message}`
+              );
+            }
+          }
+        }
       }
 
       return consolidatedValue;
@@ -1087,9 +1230,24 @@ export class EventualConsistencyPlugin extends Plugin {
   async updateAnalytics(transactions) {
     if (!this.analyticsResource || transactions.length === 0) return;
 
+    if (this.config.verbose) {
+      console.log(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Updating analytics for ${transactions.length} transactions...`
+      );
+    }
+
     try {
       // Group transactions by cohort hour
       const byHour = this._groupByCohort(transactions, 'cohortHour');
+      const cohortCount = Object.keys(byHour).length;
+
+      if (this.config.verbose) {
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Updating ${cohortCount} hourly analytics cohorts...`
+        );
+      }
 
       // Update hourly analytics
       for (const [cohort, txns] of Object.entries(byHour)) {
@@ -1099,14 +1257,31 @@ export class EventualConsistencyPlugin extends Plugin {
       // Roll up to daily and monthly if configured
       if (this.config.analyticsConfig.rollupStrategy === 'incremental') {
         const uniqueHours = Object.keys(byHour);
+
+        if (this.config.verbose) {
+          console.log(
+            `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+            `Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
+          );
+        }
+
         for (const cohortHour of uniqueHours) {
           await this._rollupAnalytics(cohortHour);
         }
       }
-    } catch (error) {
+
       if (this.config.verbose) {
-        console.warn(`[EventualConsistency] Analytics update error:`, error.message);
+        console.log(
+          `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+          `Analytics update complete for ${cohortCount} cohorts`
+        );
       }
+    } catch (error) {
+      console.warn(
+        `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
+        `Analytics update error:`,
+        error.message
+      );
     }
   }
 

package/src/resource.class.js

@@ -136,7 +136,8 @@ export class Resource extends AsyncEventEmitter {
       versioningEnabled = false,
       events = {},
       asyncEvents = true,
-      asyncPartitions = true
+      asyncPartitions = true,
+      createdBy = 'user'
     } = config;
 
     // Set instance properties
@@ -179,6 +180,7 @@ export class Resource extends AsyncEventEmitter {
       allNestedObjectsOptional,
       asyncEvents,
       asyncPartitions,
+      createdBy,
     };
 
     // Initialize hooks system