s3db.js 13.4.0 → 13.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +25 -10
  2. package/dist/{s3db.cjs.js → s3db.cjs} +38801 -32446
  3. package/dist/s3db.cjs.map +1 -0
  4. package/dist/s3db.es.js +38653 -32291
  5. package/dist/s3db.es.js.map +1 -1
  6. package/package.json +218 -22
  7. package/src/concerns/id.js +90 -6
  8. package/src/concerns/index.js +2 -1
  9. package/src/concerns/password-hashing.js +150 -0
  10. package/src/database.class.js +6 -2
  11. package/src/plugins/api/auth/basic-auth.js +40 -10
  12. package/src/plugins/api/auth/index.js +49 -3
  13. package/src/plugins/api/auth/oauth2-auth.js +171 -0
  14. package/src/plugins/api/auth/oidc-auth.js +789 -0
  15. package/src/plugins/api/auth/oidc-client.js +462 -0
  16. package/src/plugins/api/auth/path-auth-matcher.js +284 -0
  17. package/src/plugins/api/concerns/event-emitter.js +134 -0
  18. package/src/plugins/api/concerns/failban-manager.js +651 -0
  19. package/src/plugins/api/concerns/guards-helpers.js +402 -0
  20. package/src/plugins/api/concerns/metrics-collector.js +346 -0
  21. package/src/plugins/api/index.js +510 -57
  22. package/src/plugins/api/middlewares/failban.js +305 -0
  23. package/src/plugins/api/middlewares/rate-limit.js +301 -0
  24. package/src/plugins/api/middlewares/request-id.js +74 -0
  25. package/src/plugins/api/middlewares/security-headers.js +120 -0
  26. package/src/plugins/api/middlewares/session-tracking.js +194 -0
  27. package/src/plugins/api/routes/auth-routes.js +119 -78
  28. package/src/plugins/api/routes/resource-routes.js +73 -30
  29. package/src/plugins/api/server.js +1139 -45
  30. package/src/plugins/api/utils/custom-routes.js +102 -0
  31. package/src/plugins/api/utils/guards.js +213 -0
  32. package/src/plugins/api/utils/mime-types.js +154 -0
  33. package/src/plugins/api/utils/openapi-generator.js +91 -12
  34. package/src/plugins/api/utils/path-matcher.js +173 -0
  35. package/src/plugins/api/utils/static-filesystem.js +262 -0
  36. package/src/plugins/api/utils/static-s3.js +231 -0
  37. package/src/plugins/api/utils/template-engine.js +188 -0
  38. package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +853 -0
  39. package/src/plugins/cloud-inventory/drivers/aws-driver.js +2554 -0
  40. package/src/plugins/cloud-inventory/drivers/azure-driver.js +637 -0
  41. package/src/plugins/cloud-inventory/drivers/base-driver.js +99 -0
  42. package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +620 -0
  43. package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +698 -0
  44. package/src/plugins/cloud-inventory/drivers/gcp-driver.js +645 -0
  45. package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +559 -0
  46. package/src/plugins/cloud-inventory/drivers/linode-driver.js +614 -0
  47. package/src/plugins/cloud-inventory/drivers/mock-drivers.js +449 -0
  48. package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +771 -0
  49. package/src/plugins/cloud-inventory/drivers/oracle-driver.js +768 -0
  50. package/src/plugins/cloud-inventory/drivers/vultr-driver.js +636 -0
  51. package/src/plugins/cloud-inventory/index.js +20 -0
  52. package/src/plugins/cloud-inventory/registry.js +146 -0
  53. package/src/plugins/cloud-inventory/terraform-exporter.js +362 -0
  54. package/src/plugins/cloud-inventory.plugin.js +1333 -0
  55. package/src/plugins/concerns/plugin-dependencies.js +62 -2
  56. package/src/plugins/eventual-consistency/analytics.js +1 -0
  57. package/src/plugins/eventual-consistency/consolidation.js +2 -2
  58. package/src/plugins/eventual-consistency/garbage-collection.js +2 -2
  59. package/src/plugins/eventual-consistency/install.js +2 -2
  60. package/src/plugins/identity/README.md +335 -0
  61. package/src/plugins/identity/concerns/mfa-manager.js +204 -0
  62. package/src/plugins/identity/concerns/password.js +138 -0
  63. package/src/plugins/identity/concerns/resource-schemas.js +273 -0
  64. package/src/plugins/identity/concerns/token-generator.js +172 -0
  65. package/src/plugins/identity/email-service.js +422 -0
  66. package/src/plugins/identity/index.js +1052 -0
  67. package/src/plugins/identity/oauth2-server.js +1033 -0
  68. package/src/plugins/identity/oidc-discovery.js +285 -0
  69. package/src/plugins/identity/rsa-keys.js +323 -0
  70. package/src/plugins/identity/server.js +500 -0
  71. package/src/plugins/identity/session-manager.js +453 -0
  72. package/src/plugins/identity/ui/layouts/base.js +251 -0
  73. package/src/plugins/identity/ui/middleware.js +135 -0
  74. package/src/plugins/identity/ui/pages/admin/client-form.js +247 -0
  75. package/src/plugins/identity/ui/pages/admin/clients.js +179 -0
  76. package/src/plugins/identity/ui/pages/admin/dashboard.js +181 -0
  77. package/src/plugins/identity/ui/pages/admin/user-form.js +283 -0
  78. package/src/plugins/identity/ui/pages/admin/users.js +263 -0
  79. package/src/plugins/identity/ui/pages/consent.js +262 -0
  80. package/src/plugins/identity/ui/pages/forgot-password.js +104 -0
  81. package/src/plugins/identity/ui/pages/login.js +144 -0
  82. package/src/plugins/identity/ui/pages/mfa-backup-codes.js +180 -0
  83. package/src/plugins/identity/ui/pages/mfa-enrollment.js +187 -0
  84. package/src/plugins/identity/ui/pages/mfa-verification.js +178 -0
  85. package/src/plugins/identity/ui/pages/oauth-error.js +225 -0
  86. package/src/plugins/identity/ui/pages/profile.js +361 -0
  87. package/src/plugins/identity/ui/pages/register.js +226 -0
  88. package/src/plugins/identity/ui/pages/reset-password.js +128 -0
  89. package/src/plugins/identity/ui/pages/verify-email.js +172 -0
  90. package/src/plugins/identity/ui/routes.js +2541 -0
  91. package/src/plugins/identity/ui/styles/main.css +465 -0
  92. package/src/plugins/index.js +4 -1
  93. package/src/plugins/ml/base-model.class.js +65 -16
  94. package/src/plugins/ml/classification-model.class.js +1 -1
  95. package/src/plugins/ml/timeseries-model.class.js +3 -1
  96. package/src/plugins/ml.plugin.js +584 -31
  97. package/src/plugins/shared/error-handler.js +147 -0
  98. package/src/plugins/shared/index.js +9 -0
  99. package/src/plugins/shared/middlewares/compression.js +117 -0
  100. package/src/plugins/shared/middlewares/cors.js +49 -0
  101. package/src/plugins/shared/middlewares/index.js +11 -0
  102. package/src/plugins/shared/middlewares/logging.js +54 -0
  103. package/src/plugins/shared/middlewares/rate-limit.js +73 -0
  104. package/src/plugins/shared/middlewares/security.js +158 -0
  105. package/src/plugins/shared/response-formatter.js +264 -0
  106. package/src/plugins/state-machine.plugin.js +57 -2
  107. package/src/resource.class.js +140 -12
  108. package/src/schema.class.js +30 -1
  109. package/src/validator.class.js +57 -6
  110. package/dist/s3db.cjs.js.map +0 -1
@@ -6,6 +6,7 @@
6
6
  */
7
7
 
8
8
  import { Plugin } from './plugin.class.js';
9
+ import { Resource } from '../resource.class.js';
9
10
  import { requirePluginDependency } from './concerns/plugin-dependencies.js';
10
11
  import tryFn from '../concerns/try-fn.js';
11
12
 
@@ -71,12 +72,12 @@ export class MLPlugin extends Plugin {
71
72
  enableVersioning: options.enableVersioning !== false // Default true
72
73
  };
73
74
 
74
- // Validate TensorFlow.js dependency
75
- requirePluginDependency('ml-plugin');
76
-
77
75
  // Model instances
78
76
  this.models = {};
79
77
 
78
+ // Dependency validation flag (lazy validation)
79
+ this._dependenciesValidated = false;
80
+
80
81
  // Model versioning
81
82
  this.modelVersions = new Map(); // Track versions per model: { currentVersion, latestVersion }
82
83
 
@@ -86,6 +87,8 @@ export class MLPlugin extends Plugin {
86
87
  // Training state
87
88
  this.training = new Map(); // Track ongoing training
88
89
  this.insertCounters = new Map(); // Track inserts per resource
90
+ this._pendingAutoTrainingHandlers = new Map();
91
+ this._autoTrainingInitialized = new Set();
89
92
 
90
93
  // Interval handles for auto-training
91
94
  this.intervals = [];
@@ -107,6 +110,41 @@ export class MLPlugin extends Plugin {
107
110
  console.log('[MLPlugin] Installing ML Plugin...');
108
111
  }
109
112
 
113
+ // Validate plugin dependencies (lazy validation)
114
+ if (!this._dependenciesValidated) {
115
+ // Try direct import first (works better with Jest ESM)
116
+ let tfAvailable = false;
117
+ try {
118
+ await import('@tensorflow/tfjs-node');
119
+ tfAvailable = true;
120
+ if (this.config.verbose) {
121
+ console.log('[MLPlugin] TensorFlow.js loaded successfully');
122
+ }
123
+ } catch (directImportErr) {
124
+ // Fallback to plugin dependency check
125
+ const result = await requirePluginDependency('ml-plugin', {
126
+ throwOnError: false,
127
+ checkVersions: true
128
+ });
129
+
130
+ if (!result.valid) {
131
+ throw new TensorFlowDependencyError(
132
+ 'TensorFlow.js dependency not found. Install with: pnpm add @tensorflow/tfjs-node\n' +
133
+ result.messages.join('\n')
134
+ );
135
+ }
136
+ tfAvailable = result.valid;
137
+ }
138
+
139
+ if (!tfAvailable) {
140
+ throw new TensorFlowDependencyError(
141
+ 'TensorFlow.js dependency not found. Install with: pnpm add @tensorflow/tfjs-node'
142
+ );
143
+ }
144
+
145
+ this._dependenciesValidated = true;
146
+ }
147
+
110
148
  // Validate model configurations
111
149
  for (const [modelName, modelConfig] of Object.entries(this.config.models)) {
112
150
  this._validateModelConfig(modelName, modelConfig);
@@ -180,6 +218,13 @@ export class MLPlugin extends Plugin {
180
218
  }
181
219
  }
182
220
 
221
+ // Remove pending auto-training handlers
222
+ for (const handler of this._pendingAutoTrainingHandlers.values()) {
223
+ this.database.off('db:resource-created', handler);
224
+ }
225
+ this._pendingAutoTrainingHandlers.clear();
226
+ this._autoTrainingInitialized.clear();
227
+
183
228
  if (this.config.verbose) {
184
229
  console.log('[MLPlugin] Stopped');
185
230
  }
@@ -231,10 +276,166 @@ export class MLPlugin extends Plugin {
231
276
  this.database._mlPlugin = this;
232
277
  }
233
278
 
279
+ // Create namespace "ml" on Resource prototype
280
+ if (!Object.prototype.hasOwnProperty.call(Resource.prototype, 'ml')) {
281
+ Object.defineProperty(Resource.prototype, 'ml', {
282
+ get() {
283
+ const resource = this;
284
+ const mlPlugin = resource.database?._mlPlugin;
285
+
286
+ if (!mlPlugin) {
287
+ throw new Error('MLPlugin not installed');
288
+ }
289
+
290
+ return {
291
+ /**
292
+ * Auto-setup and train ML model (zero-config)
293
+ * @param {string} target - Target attribute to predict
294
+ * @param {Object} options - Configuration options
295
+ * @returns {Promise<Object>} Training results
296
+ */
297
+ learn: async (target, options = {}) => {
298
+ return await mlPlugin._resourceLearn(resource.name, target, options);
299
+ },
300
+
301
+ /**
302
+ * Make prediction
303
+ * @param {Object} input - Input features
304
+ * @param {string} target - Target attribute
305
+ * @returns {Promise<Object>} Prediction result
306
+ */
307
+ predict: async (input, target) => {
308
+ return await mlPlugin._resourcePredict(resource.name, input, target);
309
+ },
310
+
311
+ /**
312
+ * Train model manually
313
+ * @param {string} target - Target attribute
314
+ * @param {Object} options - Training options
315
+ * @returns {Promise<Object>} Training results
316
+ */
317
+ train: async (target, options = {}) => {
318
+ return await mlPlugin._resourceTrainModel(resource.name, target, options);
319
+ },
320
+
321
+ /**
322
+ * List all models for this resource
323
+ * @returns {Array} List of models
324
+ */
325
+ list: () => {
326
+ return mlPlugin._resourceListModels(resource.name);
327
+ },
328
+
329
+ /**
330
+ * List model versions
331
+ * @param {string} target - Target attribute
332
+ * @returns {Promise<Array>} List of versions
333
+ */
334
+ versions: async (target) => {
335
+ const modelName = mlPlugin._findModelForResource(resource.name, target);
336
+ if (!modelName) {
337
+ throw new ModelNotFoundError(
338
+ `No model found for resource "${resource.name}" with target "${target}"`,
339
+ { resourceName: resource.name, targetAttribute: target }
340
+ );
341
+ }
342
+ return await mlPlugin.listModelVersions(modelName);
343
+ },
344
+
345
+ /**
346
+ * Rollback to previous version
347
+ * @param {string} target - Target attribute
348
+ * @param {number} version - Version to rollback to (optional)
349
+ * @returns {Promise<Object>} Rollback info
350
+ */
351
+ rollback: async (target, version = null) => {
352
+ const modelName = mlPlugin._findModelForResource(resource.name, target);
353
+ if (!modelName) {
354
+ throw new ModelNotFoundError(
355
+ `No model found for resource "${resource.name}" with target "${target}"`,
356
+ { resourceName: resource.name, targetAttribute: target }
357
+ );
358
+ }
359
+ return await mlPlugin.rollbackVersion(modelName, version);
360
+ },
361
+
362
+ /**
363
+ * Compare two versions
364
+ * @param {string} target - Target attribute
365
+ * @param {number} v1 - First version
366
+ * @param {number} v2 - Second version
367
+ * @returns {Promise<Object>} Comparison results
368
+ */
369
+ compare: async (target, v1, v2) => {
370
+ const modelName = mlPlugin._findModelForResource(resource.name, target);
371
+ if (!modelName) {
372
+ throw new ModelNotFoundError(
373
+ `No model found for resource "${resource.name}" with target "${target}"`,
374
+ { resourceName: resource.name, targetAttribute: target }
375
+ );
376
+ }
377
+ return await mlPlugin.compareVersions(modelName, v1, v2);
378
+ },
379
+
380
+ /**
381
+ * Get model statistics
382
+ * @param {string} target - Target attribute
383
+ * @returns {Object} Model stats
384
+ */
385
+ stats: (target) => {
386
+ const modelName = mlPlugin._findModelForResource(resource.name, target);
387
+ if (!modelName) {
388
+ throw new ModelNotFoundError(
389
+ `No model found for resource "${resource.name}" with target "${target}"`,
390
+ { resourceName: resource.name, targetAttribute: target }
391
+ );
392
+ }
393
+ return mlPlugin.getModelStats(modelName);
394
+ },
395
+
396
+ /**
397
+ * Export model
398
+ * @param {string} target - Target attribute
399
+ * @returns {Promise<Object>} Exported model
400
+ */
401
+ export: async (target) => {
402
+ const modelName = mlPlugin._findModelForResource(resource.name, target);
403
+ if (!modelName) {
404
+ throw new ModelNotFoundError(
405
+ `No model found for resource "${resource.name}" with target "${target}"`,
406
+ { resourceName: resource.name, targetAttribute: target }
407
+ );
408
+ }
409
+ return await mlPlugin.exportModel(modelName);
410
+ },
411
+
412
+ /**
413
+ * Import model
414
+ * @param {string} target - Target attribute
415
+ * @param {Object} data - Model data
416
+ * @returns {Promise<void>}
417
+ */
418
+ import: async (target, data) => {
419
+ const modelName = mlPlugin._findModelForResource(resource.name, target);
420
+ if (!modelName) {
421
+ throw new ModelNotFoundError(
422
+ `No model found for resource "${resource.name}" with target "${target}"`,
423
+ { resourceName: resource.name, targetAttribute: target }
424
+ );
425
+ }
426
+ return await mlPlugin.importModel(modelName, data);
427
+ }
428
+ };
429
+ },
430
+ configurable: true
431
+ });
432
+ }
433
+
434
+ // Keep legacy methods for backward compatibility
234
435
  // Add predict() method to Resource prototype
235
- if (!this.database.Resource.prototype.predict) {
236
- this.database.Resource.prototype.predict = async function(input, targetAttribute) {
237
- const mlPlugin = this.database._mlPlugin;
436
+ if (!Object.prototype.hasOwnProperty.call(Resource.prototype, 'predict')) {
437
+ Resource.prototype.predict = async function(input, targetAttribute) {
438
+ const mlPlugin = this.database?._mlPlugin;
238
439
  if (!mlPlugin) {
239
440
  throw new Error('MLPlugin not installed');
240
441
  }
@@ -244,9 +445,9 @@ export class MLPlugin extends Plugin {
244
445
  }
245
446
 
246
447
  // Add trainModel() method to Resource prototype
247
- if (!this.database.Resource.prototype.trainModel) {
248
- this.database.Resource.prototype.trainModel = async function(targetAttribute, options = {}) {
249
- const mlPlugin = this.database._mlPlugin;
448
+ if (!Object.prototype.hasOwnProperty.call(Resource.prototype, 'trainModel')) {
449
+ Resource.prototype.trainModel = async function(targetAttribute, options = {}) {
450
+ const mlPlugin = this.database?._mlPlugin;
250
451
  if (!mlPlugin) {
251
452
  throw new Error('MLPlugin not installed');
252
453
  }
@@ -256,9 +457,9 @@ export class MLPlugin extends Plugin {
256
457
  }
257
458
 
258
459
  // Add listModels() method to Resource prototype
259
- if (!this.database.Resource.prototype.listModels) {
260
- this.database.Resource.prototype.listModels = function() {
261
- const mlPlugin = this.database._mlPlugin;
460
+ if (!Object.prototype.hasOwnProperty.call(Resource.prototype, 'listModels')) {
461
+ Resource.prototype.listModels = function() {
462
+ const mlPlugin = this.database?._mlPlugin;
262
463
  if (!mlPlugin) {
263
464
  throw new Error('MLPlugin not installed');
264
465
  }
@@ -268,7 +469,7 @@ export class MLPlugin extends Plugin {
268
469
  }
269
470
 
270
471
  if (this.config.verbose) {
271
- console.log('[MLPlugin] Injected ML methods into Resource prototype');
472
+ console.log('[MLPlugin] Injected ML namespace (resource.ml.*) into Resource prototype');
272
473
  }
273
474
  }
274
475
 
@@ -296,6 +497,283 @@ export class MLPlugin extends Plugin {
296
497
  return null;
297
498
  }
298
499
 
500
+ /**
501
+ * Auto-setup and train ML model (resource.ml.learn implementation)
502
+ * @param {string} resourceName - Resource name
503
+ * @param {string} target - Target attribute to predict
504
+ * @param {Object} options - Configuration options
505
+ * @returns {Promise<Object>} Training results
506
+ * @private
507
+ */
508
+ async _resourceLearn(resourceName, target, options = {}) {
509
+ // Check if model already exists
510
+ let modelName = this._findModelForResource(resourceName, target);
511
+
512
+ if (modelName) {
513
+ // Model exists, just retrain
514
+ if (this.config.verbose) {
515
+ console.log(`[MLPlugin] Model "${modelName}" already exists, retraining...`);
516
+ }
517
+ return await this.train(modelName, options);
518
+ }
519
+
520
+ // Create new model dynamically
521
+ modelName = `${resourceName}_${target}_auto`;
522
+
523
+ if (this.config.verbose) {
524
+ console.log(`[MLPlugin] Auto-creating model "${modelName}" for ${resourceName}.${target}...`);
525
+ }
526
+
527
+ // Get resource
528
+ const resource = this.database.resources[resourceName];
529
+ if (!resource) {
530
+ throw new ModelConfigError(
531
+ `Resource "${resourceName}" not found`,
532
+ { resourceName, availableResources: Object.keys(this.database.resources) }
533
+ );
534
+ }
535
+
536
+ // Auto-detect type if not specified
537
+ let modelType = options.type;
538
+ if (!modelType) {
539
+ modelType = await this._autoDetectType(resourceName, target);
540
+ if (this.config.verbose) {
541
+ console.log(`[MLPlugin] Auto-detected type: ${modelType}`);
542
+ }
543
+ }
544
+
545
+ // Auto-select features if not specified
546
+ let features = options.features;
547
+ if (!features || features.length === 0) {
548
+ features = await this._autoSelectFeatures(resourceName, target);
549
+ if (this.config.verbose) {
550
+ console.log(`[MLPlugin] Auto-selected features: ${features.join(', ')}`);
551
+ }
552
+ }
553
+
554
+ // Get sample count to adjust batchSize automatically
555
+ const [samplesOk, samplesErr, sampleData] = await tryFn(() => resource.list());
556
+ const sampleCount = (samplesOk && sampleData) ? sampleData.length : 0;
557
+
558
+ // Get default model config and adjust batchSize based on available data
559
+ let defaultModelConfig = this._getDefaultModelConfig(modelType);
560
+
561
+ // Check if user explicitly provided batchSize
562
+ const userProvidedBatchSize = options.modelConfig && options.modelConfig.batchSize !== undefined;
563
+
564
+ if (!userProvidedBatchSize && sampleCount > 0 && sampleCount < defaultModelConfig.batchSize) {
565
+ // Adjust batchSize to be at most half of available samples (only if user didn't provide one)
566
+ defaultModelConfig.batchSize = Math.max(4, Math.floor(sampleCount / 2));
567
+ if (this.config.verbose) {
568
+ console.log(`[MLPlugin] Auto-adjusted batchSize to ${defaultModelConfig.batchSize} based on ${sampleCount} samples`);
569
+ }
570
+ }
571
+
572
+ // Merge custom modelConfig with defaults
573
+ // If user didn't provide batchSize, keep the auto-adjusted one from defaultModelConfig
574
+ const customModelConfig = options.modelConfig || {};
575
+ const mergedModelConfig = {
576
+ ...defaultModelConfig,
577
+ ...customModelConfig,
578
+ // Preserve auto-adjusted batchSize if user didn't provide one
579
+ ...(!userProvidedBatchSize && { batchSize: defaultModelConfig.batchSize })
580
+ };
581
+
582
+ // Create model config
583
+ const modelConfig = {
584
+ type: modelType,
585
+ resource: resourceName,
586
+ features: features,
587
+ target: target,
588
+ autoTrain: options.autoTrain !== undefined ? options.autoTrain : false,
589
+ saveModel: options.saveModel !== undefined ? options.saveModel : true,
590
+ saveTrainingData: options.saveTrainingData !== undefined ? options.saveTrainingData : false,
591
+ modelConfig: mergedModelConfig,
592
+ ...options
593
+ };
594
+
595
+ // Register model
596
+ this.config.models[modelName] = modelConfig;
597
+
598
+ // Initialize model
599
+ await this._initializeModel(modelName, modelConfig);
600
+
601
+ // Update cache
602
+ this._buildModelCache();
603
+
604
+ // Train immediately
605
+ if (this.config.verbose) {
606
+ console.log(`[MLPlugin] Training model "${modelName}"...`);
607
+ }
608
+
609
+ const result = await this.train(modelName, options);
610
+
611
+ if (this.config.verbose) {
612
+ console.log(`[MLPlugin] ✅ Model "${modelName}" ready!`);
613
+ }
614
+
615
+ return {
616
+ modelName,
617
+ type: modelType,
618
+ features,
619
+ target,
620
+ ...result
621
+ };
622
+ }
623
+
624
+ /**
625
+ * Auto-detect model type based on target attribute
626
+ * @param {string} resourceName - Resource name
627
+ * @param {string} target - Target attribute
628
+ * @returns {Promise<string>} Model type
629
+ * @private
630
+ */
631
+ async _autoDetectType(resourceName, target) {
632
+ const resource = this.database.resources[resourceName];
633
+
634
+ // Get some sample data
635
+ const [ok, err, samples] = await tryFn(() => resource.list({ limit: 100 }));
636
+
637
+ if (!ok || !samples || samples.length === 0) {
638
+ // Default to regression if no data
639
+ return 'regression';
640
+ }
641
+
642
+ // Analyze target values
643
+ const targetValues = samples.map(s => s[target]).filter(v => v != null);
644
+
645
+ if (targetValues.length === 0) {
646
+ return 'regression';
647
+ }
648
+
649
+ // Check if numeric
650
+ const isNumeric = targetValues.every(v => typeof v === 'number');
651
+
652
+ if (isNumeric) {
653
+ // Check for time series (if data has timestamp)
654
+ const hasTimestamp = samples.every(s => s.timestamp || s.createdAt || s.date);
655
+ if (hasTimestamp) {
656
+ return 'timeseries';
657
+ }
658
+ return 'regression';
659
+ }
660
+
661
+ // Check if categorical (strings/booleans)
662
+ const isCategorical = targetValues.every(v => typeof v === 'string' || typeof v === 'boolean');
663
+
664
+ if (isCategorical) {
665
+ return 'classification';
666
+ }
667
+
668
+ // Default
669
+ return 'regression';
670
+ }
671
+
672
+ /**
673
+ * Auto-select best features for prediction
674
+ * @param {string} resourceName - Resource name
675
+ * @param {string} target - Target attribute
676
+ * @returns {Promise<Array>} Selected features
677
+ * @private
678
+ */
679
+ async _autoSelectFeatures(resourceName, target) {
680
+ const resource = this.database.resources[resourceName];
681
+
682
+ // Get all numeric attributes from schema
683
+ const schema = resource.schema;
684
+ const attributes = schema?.attributes || {};
685
+
686
+ const numericFields = [];
687
+
688
+ for (const [fieldName, fieldDef] of Object.entries(attributes)) {
689
+ // Skip target
690
+ if (fieldName === target) continue;
691
+
692
+ // Skip system fields
693
+ if (['id', 'createdAt', 'updatedAt', 'createdBy'].includes(fieldName)) continue;
694
+
695
+ // Check if numeric type
696
+ const fieldType = typeof fieldDef === 'string' ? fieldDef.split('|')[0] : fieldDef.type;
697
+
698
+ if (fieldType === 'number' || fieldType === 'integer' || fieldType === 'float') {
699
+ numericFields.push(fieldName);
700
+ }
701
+ }
702
+
703
+ // If no numeric fields found, try to detect from data
704
+ if (numericFields.length === 0) {
705
+ const [ok, err, samples] = await tryFn(() => resource.list({ limit: 10 }));
706
+
707
+ if (ok && samples && samples.length > 0) {
708
+ const firstSample = samples[0];
709
+
710
+ for (const [key, value] of Object.entries(firstSample)) {
711
+ if (key === target) continue;
712
+ if (['id', 'createdAt', 'updatedAt', 'createdBy'].includes(key)) continue;
713
+
714
+ if (typeof value === 'number') {
715
+ numericFields.push(key);
716
+ }
717
+ }
718
+ }
719
+ }
720
+
721
+ if (numericFields.length === 0) {
722
+ throw new ModelConfigError(
723
+ `No numeric features found for target "${target}" in resource "${resourceName}"`,
724
+ { resourceName, target, availableAttributes: Object.keys(attributes) }
725
+ );
726
+ }
727
+
728
+ return numericFields;
729
+ }
730
+
731
+ /**
732
+ * Get default model config for type
733
+ * @param {string} type - Model type
734
+ * @returns {Object} Default config
735
+ * @private
736
+ */
737
+ _getDefaultModelConfig(type) {
738
+ const defaults = {
739
+ regression: {
740
+ epochs: 50,
741
+ batchSize: 32,
742
+ learningRate: 0.01,
743
+ validationSplit: 0.2,
744
+ polynomial: 1
745
+ },
746
+ classification: {
747
+ epochs: 50,
748
+ batchSize: 32,
749
+ learningRate: 0.01,
750
+ validationSplit: 0.2,
751
+ units: 64,
752
+ dropout: 0.2
753
+ },
754
+ timeseries: {
755
+ epochs: 50,
756
+ batchSize: 16,
757
+ learningRate: 0.001,
758
+ validationSplit: 0.2,
759
+ lookback: 10,
760
+ lstmUnits: 50
761
+ },
762
+ 'neural-network': {
763
+ epochs: 50,
764
+ batchSize: 32,
765
+ learningRate: 0.01,
766
+ validationSplit: 0.2,
767
+ layers: [
768
+ { units: 64, activation: 'relu', dropout: 0.2 },
769
+ { units: 32, activation: 'relu' }
770
+ ]
771
+ }
772
+ };
773
+
774
+ return defaults[type] || defaults.regression;
775
+ }
776
+
299
777
  /**
300
778
  * Resource predict implementation
301
779
  * @private
@@ -406,6 +884,7 @@ export class MLPlugin extends Plugin {
406
884
  resource: config.resource,
407
885
  features: config.features,
408
886
  target: config.target,
887
+ minSamples: config.minSamples ?? this.config.minTrainingSamples,
409
888
  modelConfig: config.modelConfig || {},
410
889
  verbose: this.config.verbose
411
890
  };
@@ -447,17 +926,39 @@ export class MLPlugin extends Plugin {
447
926
  /**
448
927
  * Setup auto-training for a model
449
928
  * @private
450
- */
929
+ */
451
930
  _setupAutoTraining(modelName, config) {
931
+ if (!this.insertCounters.has(modelName)) {
932
+ this.insertCounters.set(modelName, 0);
933
+ }
934
+
452
935
  const resource = this.database.resources[config.resource];
453
936
 
454
937
  if (!resource) {
455
- console.warn(`[MLPlugin] Resource "${config.resource}" not found for model "${modelName}"`);
938
+ if (this.config.verbose) {
939
+ console.warn(`[MLPlugin] Resource "${config.resource}" not found for model "${modelName}". Auto-training will attach when resource is created.`);
940
+ }
941
+
942
+ if (!this._pendingAutoTrainingHandlers.has(modelName)) {
943
+ const handler = (createdName) => {
944
+ if (createdName !== config.resource) {
945
+ return;
946
+ }
947
+
948
+ this.database.off('db:resource-created', handler);
949
+ this._pendingAutoTrainingHandlers.delete(modelName);
950
+ this._setupAutoTraining(modelName, config);
951
+ };
952
+
953
+ this._pendingAutoTrainingHandlers.set(modelName, handler);
954
+ this.database.on('db:resource-created', handler);
955
+ }
456
956
  return;
457
957
  }
458
958
 
459
- // Initialize insert counter
460
- this.insertCounters.set(modelName, 0);
959
+ if (this._autoTrainingInitialized.has(modelName)) {
960
+ return;
961
+ }
461
962
 
462
963
  // Hook: Track inserts
463
964
  if (config.trainAfterInserts && config.trainAfterInserts > 0) {
@@ -507,6 +1008,8 @@ export class MLPlugin extends Plugin {
507
1008
  console.log(`[MLPlugin] Setup interval training for "${modelName}" (every ${config.trainInterval}ms)`);
508
1009
  }
509
1010
  }
1011
+
1012
+ this._autoTrainingInitialized.add(modelName);
510
1013
  }
511
1014
 
512
1015
  /**
@@ -563,7 +1066,10 @@ export class MLPlugin extends Plugin {
563
1066
  }
564
1067
 
565
1068
  const [ok, err, partitionData] = await tryFn(() =>
566
- resource.listPartition(partition.name, partition.values)
1069
+ resource.listPartition({
1070
+ partition: partition.name,
1071
+ partitionValues: partition.values
1072
+ })
567
1073
  );
568
1074
 
569
1075
  if (!ok) {
@@ -923,12 +1429,13 @@ export class MLPlugin extends Plugin {
923
1429
  return;
924
1430
  }
925
1431
 
1432
+ const modelStats = this.models[modelName].getStats();
1433
+ const timestamp = new Date().toISOString();
926
1434
  const enableVersioning = this.config.enableVersioning;
927
1435
 
928
1436
  if (enableVersioning) {
929
1437
  // Save with version
930
1438
  const version = this._getNextVersion(modelName);
931
- const modelStats = this.models[modelName].getStats();
932
1439
 
933
1440
  // Save versioned model binary to S3 body
934
1441
  await storage.set(
@@ -943,7 +1450,7 @@ export class MLPlugin extends Plugin {
943
1450
  accuracy: modelStats.accuracy,
944
1451
  samples: modelStats.samples
945
1452
  },
946
- savedAt: new Date().toISOString()
1453
+ savedAt: timestamp
947
1454
  },
948
1455
  { behavior: 'body-only' } // Large binary data goes to S3 body
949
1456
  );
@@ -958,7 +1465,7 @@ export class MLPlugin extends Plugin {
958
1465
  modelName,
959
1466
  version,
960
1467
  type: 'reference',
961
- updatedAt: new Date().toISOString()
1468
+ updatedAt: timestamp
962
1469
  },
963
1470
  { behavior: 'body-overflow' } // Small metadata
964
1471
  );
@@ -974,7 +1481,12 @@ export class MLPlugin extends Plugin {
974
1481
  modelName,
975
1482
  type: 'model',
976
1483
  modelData: exportedModel,
977
- savedAt: new Date().toISOString()
1484
+ metrics: {
1485
+ loss: modelStats.loss,
1486
+ accuracy: modelStats.accuracy,
1487
+ samples: modelStats.samples
1488
+ },
1489
+ savedAt: timestamp
978
1490
  },
979
1491
  { behavior: 'body-only' }
980
1492
  );
@@ -983,6 +1495,34 @@ export class MLPlugin extends Plugin {
983
1495
  console.log(`[MLPlugin] Saved model "${modelName}" to S3 (resource=${resourceName}/plugin=ml/models/${modelName}/latest)`);
984
1496
  }
985
1497
  }
1498
+
1499
+ // Legacy compatibility record (flat key: model_{modelName})
1500
+ const activeVersion = enableVersioning
1501
+ ? (this.modelVersions.get(modelName)?.latestVersion || 1)
1502
+ : undefined;
1503
+
1504
+ const compatibilityData = enableVersioning
1505
+ ? {
1506
+ storageKey: storage.getPluginKey(resourceName, 'models', modelName, `v${activeVersion}`),
1507
+ version: activeVersion
1508
+ }
1509
+ : exportedModel;
1510
+
1511
+ await storage.set(
1512
+ `model_${modelName}`,
1513
+ {
1514
+ modelName,
1515
+ type: 'model',
1516
+ data: compatibilityData,
1517
+ metrics: {
1518
+ loss: modelStats.loss,
1519
+ accuracy: modelStats.accuracy,
1520
+ samples: modelStats.samples
1521
+ },
1522
+ savedAt: timestamp
1523
+ },
1524
+ { behavior: enableVersioning ? 'body-overflow' : 'body-only' }
1525
+ );
986
1526
  } catch (error) {
987
1527
  console.error(`[MLPlugin] Failed to save model "${modelName}":`, error.message);
988
1528
  }
@@ -1220,12 +1760,14 @@ export class MLPlugin extends Plugin {
1220
1760
  return null;
1221
1761
  }
1222
1762
 
1763
+ const samplesArray = Array.isArray(record.samples) ? record.samples : [];
1764
+
1223
1765
  return {
1224
1766
  modelName: record.modelName,
1225
- samples: record.samples,
1767
+ samples: samplesArray.length,
1226
1768
  features: record.features,
1227
1769
  target: record.target,
1228
- data: record.samples,
1770
+ data: samplesArray,
1229
1771
  savedAt: record.savedAt
1230
1772
  };
1231
1773
  }
@@ -1242,11 +1784,15 @@ export class MLPlugin extends Plugin {
1242
1784
  return null;
1243
1785
  }
1244
1786
 
1787
+ const historyEntries = Array.isArray(historyData.history)
1788
+ ? historyData.history
1789
+ : JSON.parse(historyData.history);
1790
+
1245
1791
  const targetVersion = version || historyData.latestVersion;
1246
1792
  const reconstructedSamples = [];
1247
1793
 
1248
1794
  // Load and combine all versions up to target version
1249
- for (const entry of historyData.history) {
1795
+ for (const entry of historyEntries) {
1250
1796
  if (entry.version > targetVersion) break;
1251
1797
 
1252
1798
  if (entry.storageKey && entry.newSamples > 0) {
@@ -1260,15 +1806,16 @@ export class MLPlugin extends Plugin {
1260
1806
  }
1261
1807
  }
1262
1808
 
1263
- const targetEntry = historyData.history.find(e => e.version === targetVersion);
1809
+ const targetEntry = historyEntries.find(e => e.version === targetVersion);
1264
1810
 
1265
1811
  return {
1266
1812
  modelName,
1267
1813
  version: targetVersion,
1268
- samples: reconstructedSamples,
1814
+ samples: reconstructedSamples.length,
1269
1815
  totalSamples: reconstructedSamples.length,
1270
1816
  features: modelConfig.features,
1271
1817
  target: modelConfig.target,
1818
+ data: reconstructedSamples,
1272
1819
  metrics: targetEntry?.metrics,
1273
1820
  savedAt: targetEntry?.trainedAt
1274
1821
  };
@@ -1441,7 +1988,9 @@ export class MLPlugin extends Plugin {
1441
1988
 
1442
1989
  return {
1443
1990
  version,
1444
- metrics: versionData.metrics ? JSON.parse(versionData.metrics) : {},
1991
+ metrics: typeof versionData.metrics === 'string'
1992
+ ? JSON.parse(versionData.metrics)
1993
+ : (versionData.metrics || {}),
1445
1994
  savedAt: versionData.savedAt
1446
1995
  };
1447
1996
  } catch (error) {
@@ -1506,11 +2055,15 @@ export class MLPlugin extends Plugin {
1506
2055
  return null;
1507
2056
  }
1508
2057
 
2058
+ const historyEntries = Array.isArray(historyData.history)
2059
+ ? historyData.history
2060
+ : JSON.parse(historyData.history);
2061
+
1509
2062
  return {
1510
2063
  modelName: historyData.modelName,
1511
2064
  totalTrainings: historyData.totalTrainings,
1512
2065
  latestVersion: historyData.latestVersion,
1513
- history: JSON.parse(historyData.history),
2066
+ history: historyEntries,
1514
2067
  updatedAt: historyData.updatedAt
1515
2068
  };
1516
2069
  } catch (error) {
@@ -1547,8 +2100,8 @@ export class MLPlugin extends Plugin {
1547
2100
  throw new MLError(`Version ${version2} not found`, { modelName, version: version2 });
1548
2101
  }
1549
2102
 
1550
- const metrics1 = v1Data.metrics ? JSON.parse(v1Data.metrics) : {};
1551
- const metrics2 = v2Data.metrics ? JSON.parse(v2Data.metrics) : {};
2103
+ const metrics1 = typeof v1Data.metrics === 'string' ? JSON.parse(v1Data.metrics) : (v1Data.metrics || {});
2104
+ const metrics2 = typeof v2Data.metrics === 'string' ? JSON.parse(v2Data.metrics) : (v2Data.metrics || {});
1552
2105
 
1553
2106
  return {
1554
2107
  modelName,