s3db.js 12.4.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +1923 -57
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +1923 -58
- package/dist/s3db.es.js.map +1 -1
- package/package.json +5 -1
- package/src/clients/memory-client.class.js +41 -24
- package/src/database.class.js +52 -17
- package/src/plugins/api/index.js +12 -9
- package/src/plugins/api/routes/resource-routes.js +78 -0
- package/src/plugins/index.js +1 -0
- package/src/plugins/ml/base-model.class.js +459 -0
- package/src/plugins/ml/classification-model.class.js +338 -0
- package/src/plugins/ml/neural-network-model.class.js +312 -0
- package/src/plugins/ml/regression-model.class.js +159 -0
- package/src/plugins/ml/timeseries-model.class.js +346 -0
- package/src/plugins/ml.errors.js +130 -0
- package/src/plugins/ml.plugin.js +655 -0
- package/src/resource.class.js +106 -34
package/dist/s3db.es.js
CHANGED
|
@@ -2993,12 +2993,6 @@ class ApiPlugin extends Plugin {
|
|
|
2993
2993
|
async _createCompressionMiddleware() {
|
|
2994
2994
|
return async (c, next) => {
|
|
2995
2995
|
await next();
|
|
2996
|
-
const acceptEncoding = c.req.header("accept-encoding") || "";
|
|
2997
|
-
if (acceptEncoding.includes("gzip")) {
|
|
2998
|
-
c.header("Content-Encoding", "gzip");
|
|
2999
|
-
} else if (acceptEncoding.includes("deflate")) {
|
|
3000
|
-
c.header("Content-Encoding", "deflate");
|
|
3001
|
-
}
|
|
3002
2996
|
};
|
|
3003
2997
|
}
|
|
3004
2998
|
/**
|
|
@@ -11903,6 +11897,1780 @@ class MetricsPlugin extends Plugin {
|
|
|
11903
11897
|
}
|
|
11904
11898
|
}
|
|
11905
11899
|
|
|
11900
|
+
class MLError extends Error {
|
|
11901
|
+
constructor(message, context = {}) {
|
|
11902
|
+
super(message);
|
|
11903
|
+
this.name = "MLError";
|
|
11904
|
+
this.context = context;
|
|
11905
|
+
if (Error.captureStackTrace) {
|
|
11906
|
+
Error.captureStackTrace(this, this.constructor);
|
|
11907
|
+
}
|
|
11908
|
+
}
|
|
11909
|
+
toJSON() {
|
|
11910
|
+
return {
|
|
11911
|
+
name: this.name,
|
|
11912
|
+
message: this.message,
|
|
11913
|
+
context: this.context,
|
|
11914
|
+
stack: this.stack
|
|
11915
|
+
};
|
|
11916
|
+
}
|
|
11917
|
+
}
|
|
11918
|
+
class ModelConfigError extends MLError {
|
|
11919
|
+
constructor(message, context = {}) {
|
|
11920
|
+
super(message, context);
|
|
11921
|
+
this.name = "ModelConfigError";
|
|
11922
|
+
}
|
|
11923
|
+
}
|
|
11924
|
+
class TrainingError extends MLError {
|
|
11925
|
+
constructor(message, context = {}) {
|
|
11926
|
+
super(message, context);
|
|
11927
|
+
this.name = "TrainingError";
|
|
11928
|
+
}
|
|
11929
|
+
}
|
|
11930
|
+
let PredictionError$1 = class PredictionError extends MLError {
|
|
11931
|
+
constructor(message, context = {}) {
|
|
11932
|
+
super(message, context);
|
|
11933
|
+
this.name = "PredictionError";
|
|
11934
|
+
}
|
|
11935
|
+
};
|
|
11936
|
+
class ModelNotFoundError extends MLError {
|
|
11937
|
+
constructor(message, context = {}) {
|
|
11938
|
+
super(message, context);
|
|
11939
|
+
this.name = "ModelNotFoundError";
|
|
11940
|
+
}
|
|
11941
|
+
}
|
|
11942
|
+
let ModelNotTrainedError$1 = class ModelNotTrainedError extends MLError {
|
|
11943
|
+
constructor(message, context = {}) {
|
|
11944
|
+
super(message, context);
|
|
11945
|
+
this.name = "ModelNotTrainedError";
|
|
11946
|
+
}
|
|
11947
|
+
};
|
|
11948
|
+
class DataValidationError extends MLError {
|
|
11949
|
+
constructor(message, context = {}) {
|
|
11950
|
+
super(message, context);
|
|
11951
|
+
this.name = "DataValidationError";
|
|
11952
|
+
}
|
|
11953
|
+
}
|
|
11954
|
+
class InsufficientDataError extends MLError {
|
|
11955
|
+
constructor(message, context = {}) {
|
|
11956
|
+
super(message, context);
|
|
11957
|
+
this.name = "InsufficientDataError";
|
|
11958
|
+
}
|
|
11959
|
+
}
|
|
11960
|
+
class TensorFlowDependencyError extends MLError {
|
|
11961
|
+
constructor(message = "TensorFlow.js is not installed. Run: pnpm add @tensorflow/tfjs-node", context = {}) {
|
|
11962
|
+
super(message, context);
|
|
11963
|
+
this.name = "TensorFlowDependencyError";
|
|
11964
|
+
}
|
|
11965
|
+
}
|
|
11966
|
+
|
|
11967
|
+
class BaseModel {
|
|
11968
|
+
constructor(config = {}) {
|
|
11969
|
+
if (this.constructor === BaseModel) {
|
|
11970
|
+
throw new Error("BaseModel is an abstract class and cannot be instantiated directly");
|
|
11971
|
+
}
|
|
11972
|
+
this.config = {
|
|
11973
|
+
name: config.name || "unnamed",
|
|
11974
|
+
resource: config.resource,
|
|
11975
|
+
features: config.features || [],
|
|
11976
|
+
target: config.target,
|
|
11977
|
+
modelConfig: {
|
|
11978
|
+
epochs: 50,
|
|
11979
|
+
batchSize: 32,
|
|
11980
|
+
learningRate: 0.01,
|
|
11981
|
+
validationSplit: 0.2,
|
|
11982
|
+
...config.modelConfig
|
|
11983
|
+
},
|
|
11984
|
+
verbose: config.verbose || false
|
|
11985
|
+
};
|
|
11986
|
+
this.model = null;
|
|
11987
|
+
this.isTrained = false;
|
|
11988
|
+
this.normalizer = {
|
|
11989
|
+
features: {},
|
|
11990
|
+
target: {}
|
|
11991
|
+
};
|
|
11992
|
+
this.stats = {
|
|
11993
|
+
trainedAt: null,
|
|
11994
|
+
samples: 0,
|
|
11995
|
+
loss: null,
|
|
11996
|
+
accuracy: null,
|
|
11997
|
+
predictions: 0,
|
|
11998
|
+
errors: 0
|
|
11999
|
+
};
|
|
12000
|
+
this._validateTensorFlow();
|
|
12001
|
+
}
|
|
12002
|
+
/**
|
|
12003
|
+
* Validate TensorFlow.js is installed
|
|
12004
|
+
* @private
|
|
12005
|
+
*/
|
|
12006
|
+
_validateTensorFlow() {
|
|
12007
|
+
try {
|
|
12008
|
+
this.tf = require("@tensorflow/tfjs-node");
|
|
12009
|
+
} catch (error) {
|
|
12010
|
+
throw new TensorFlowDependencyError(
|
|
12011
|
+
"TensorFlow.js is not installed. Run: pnpm add @tensorflow/tfjs-node",
|
|
12012
|
+
{ originalError: error.message }
|
|
12013
|
+
);
|
|
12014
|
+
}
|
|
12015
|
+
}
|
|
12016
|
+
/**
|
|
12017
|
+
* Abstract method: Build the model architecture
|
|
12018
|
+
* Must be implemented by subclasses
|
|
12019
|
+
* @abstract
|
|
12020
|
+
*/
|
|
12021
|
+
buildModel() {
|
|
12022
|
+
throw new Error("buildModel() must be implemented by subclass");
|
|
12023
|
+
}
|
|
12024
|
+
/**
|
|
12025
|
+
* Train the model with provided data
|
|
12026
|
+
* @param {Array} data - Training data records
|
|
12027
|
+
* @returns {Object} Training results
|
|
12028
|
+
*/
|
|
12029
|
+
async train(data) {
|
|
12030
|
+
try {
|
|
12031
|
+
if (!data || data.length === 0) {
|
|
12032
|
+
throw new InsufficientDataError("No training data provided", {
|
|
12033
|
+
model: this.config.name
|
|
12034
|
+
});
|
|
12035
|
+
}
|
|
12036
|
+
const minSamples = this.config.modelConfig.batchSize || 10;
|
|
12037
|
+
if (data.length < minSamples) {
|
|
12038
|
+
throw new InsufficientDataError(
|
|
12039
|
+
`Insufficient training data: ${data.length} samples (minimum: ${minSamples})`,
|
|
12040
|
+
{ model: this.config.name, samples: data.length, minimum: minSamples }
|
|
12041
|
+
);
|
|
12042
|
+
}
|
|
12043
|
+
const { xs, ys } = this._prepareData(data);
|
|
12044
|
+
if (!this.model) {
|
|
12045
|
+
this.buildModel();
|
|
12046
|
+
}
|
|
12047
|
+
const history = await this.model.fit(xs, ys, {
|
|
12048
|
+
epochs: this.config.modelConfig.epochs,
|
|
12049
|
+
batchSize: this.config.modelConfig.batchSize,
|
|
12050
|
+
validationSplit: this.config.modelConfig.validationSplit,
|
|
12051
|
+
verbose: this.config.verbose ? 1 : 0,
|
|
12052
|
+
callbacks: {
|
|
12053
|
+
onEpochEnd: (epoch, logs) => {
|
|
12054
|
+
if (this.config.verbose && epoch % 10 === 0) {
|
|
12055
|
+
console.log(`[MLPlugin] ${this.config.name} - Epoch ${epoch}: loss=${logs.loss.toFixed(4)}`);
|
|
12056
|
+
}
|
|
12057
|
+
}
|
|
12058
|
+
}
|
|
12059
|
+
});
|
|
12060
|
+
this.isTrained = true;
|
|
12061
|
+
this.stats.trainedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12062
|
+
this.stats.samples = data.length;
|
|
12063
|
+
this.stats.loss = history.history.loss[history.history.loss.length - 1];
|
|
12064
|
+
if (history.history.acc) {
|
|
12065
|
+
this.stats.accuracy = history.history.acc[history.history.acc.length - 1];
|
|
12066
|
+
}
|
|
12067
|
+
xs.dispose();
|
|
12068
|
+
ys.dispose();
|
|
12069
|
+
if (this.config.verbose) {
|
|
12070
|
+
console.log(`[MLPlugin] ${this.config.name} - Training completed:`, {
|
|
12071
|
+
samples: this.stats.samples,
|
|
12072
|
+
loss: this.stats.loss,
|
|
12073
|
+
accuracy: this.stats.accuracy
|
|
12074
|
+
});
|
|
12075
|
+
}
|
|
12076
|
+
return {
|
|
12077
|
+
loss: this.stats.loss,
|
|
12078
|
+
accuracy: this.stats.accuracy,
|
|
12079
|
+
epochs: this.config.modelConfig.epochs,
|
|
12080
|
+
samples: this.stats.samples
|
|
12081
|
+
};
|
|
12082
|
+
} catch (error) {
|
|
12083
|
+
this.stats.errors++;
|
|
12084
|
+
if (error instanceof InsufficientDataError || error instanceof DataValidationError) {
|
|
12085
|
+
throw error;
|
|
12086
|
+
}
|
|
12087
|
+
throw new TrainingError(`Training failed: ${error.message}`, {
|
|
12088
|
+
model: this.config.name,
|
|
12089
|
+
originalError: error.message
|
|
12090
|
+
});
|
|
12091
|
+
}
|
|
12092
|
+
}
|
|
12093
|
+
/**
|
|
12094
|
+
* Make a prediction with the trained model
|
|
12095
|
+
* @param {Object} input - Input features
|
|
12096
|
+
* @returns {Object} Prediction result
|
|
12097
|
+
*/
|
|
12098
|
+
async predict(input) {
|
|
12099
|
+
if (!this.isTrained) {
|
|
12100
|
+
throw new ModelNotTrainedError$1(`Model "${this.config.name}" is not trained yet`, {
|
|
12101
|
+
model: this.config.name
|
|
12102
|
+
});
|
|
12103
|
+
}
|
|
12104
|
+
try {
|
|
12105
|
+
this._validateInput(input);
|
|
12106
|
+
const features = this._extractFeatures(input);
|
|
12107
|
+
const normalizedFeatures = this._normalizeFeatures(features);
|
|
12108
|
+
const inputTensor = this.tf.tensor2d([normalizedFeatures]);
|
|
12109
|
+
const predictionTensor = this.model.predict(inputTensor);
|
|
12110
|
+
const predictionArray = await predictionTensor.data();
|
|
12111
|
+
inputTensor.dispose();
|
|
12112
|
+
predictionTensor.dispose();
|
|
12113
|
+
const prediction = this._denormalizePrediction(predictionArray[0]);
|
|
12114
|
+
this.stats.predictions++;
|
|
12115
|
+
return {
|
|
12116
|
+
prediction,
|
|
12117
|
+
confidence: this._calculateConfidence(predictionArray[0])
|
|
12118
|
+
};
|
|
12119
|
+
} catch (error) {
|
|
12120
|
+
this.stats.errors++;
|
|
12121
|
+
if (error instanceof ModelNotTrainedError$1 || error instanceof DataValidationError) {
|
|
12122
|
+
throw error;
|
|
12123
|
+
}
|
|
12124
|
+
throw new PredictionError$1(`Prediction failed: ${error.message}`, {
|
|
12125
|
+
model: this.config.name,
|
|
12126
|
+
input,
|
|
12127
|
+
originalError: error.message
|
|
12128
|
+
});
|
|
12129
|
+
}
|
|
12130
|
+
}
|
|
12131
|
+
/**
|
|
12132
|
+
* Make predictions for multiple inputs
|
|
12133
|
+
* @param {Array} inputs - Array of input objects
|
|
12134
|
+
* @returns {Array} Array of prediction results
|
|
12135
|
+
*/
|
|
12136
|
+
async predictBatch(inputs) {
|
|
12137
|
+
if (!this.isTrained) {
|
|
12138
|
+
throw new ModelNotTrainedError$1(`Model "${this.config.name}" is not trained yet`, {
|
|
12139
|
+
model: this.config.name
|
|
12140
|
+
});
|
|
12141
|
+
}
|
|
12142
|
+
const predictions = [];
|
|
12143
|
+
for (const input of inputs) {
|
|
12144
|
+
predictions.push(await this.predict(input));
|
|
12145
|
+
}
|
|
12146
|
+
return predictions;
|
|
12147
|
+
}
|
|
12148
|
+
/**
|
|
12149
|
+
* Prepare training data (extract features and target)
|
|
12150
|
+
* @private
|
|
12151
|
+
* @param {Array} data - Raw training data
|
|
12152
|
+
* @returns {Object} Prepared tensors {xs, ys}
|
|
12153
|
+
*/
|
|
12154
|
+
_prepareData(data) {
|
|
12155
|
+
const features = [];
|
|
12156
|
+
const targets = [];
|
|
12157
|
+
for (const record of data) {
|
|
12158
|
+
const missingFeatures = this.config.features.filter((f) => !(f in record));
|
|
12159
|
+
if (missingFeatures.length > 0) {
|
|
12160
|
+
throw new DataValidationError(
|
|
12161
|
+
`Missing features in training data: ${missingFeatures.join(", ")}`,
|
|
12162
|
+
{ model: this.config.name, missingFeatures, record }
|
|
12163
|
+
);
|
|
12164
|
+
}
|
|
12165
|
+
if (!(this.config.target in record)) {
|
|
12166
|
+
throw new DataValidationError(
|
|
12167
|
+
`Missing target "${this.config.target}" in training data`,
|
|
12168
|
+
{ model: this.config.name, target: this.config.target, record }
|
|
12169
|
+
);
|
|
12170
|
+
}
|
|
12171
|
+
const featureValues = this._extractFeatures(record);
|
|
12172
|
+
features.push(featureValues);
|
|
12173
|
+
targets.push(record[this.config.target]);
|
|
12174
|
+
}
|
|
12175
|
+
this._calculateNormalizer(features, targets);
|
|
12176
|
+
const normalizedFeatures = features.map((f) => this._normalizeFeatures(f));
|
|
12177
|
+
const normalizedTargets = targets.map((t) => this._normalizeTarget(t));
|
|
12178
|
+
return {
|
|
12179
|
+
xs: this.tf.tensor2d(normalizedFeatures),
|
|
12180
|
+
ys: this._prepareTargetTensor(normalizedTargets)
|
|
12181
|
+
};
|
|
12182
|
+
}
|
|
12183
|
+
/**
|
|
12184
|
+
* Prepare target tensor (can be overridden by subclasses)
|
|
12185
|
+
* @protected
|
|
12186
|
+
* @param {Array} targets - Normalized target values
|
|
12187
|
+
* @returns {Tensor} Target tensor
|
|
12188
|
+
*/
|
|
12189
|
+
_prepareTargetTensor(targets) {
|
|
12190
|
+
return this.tf.tensor2d(targets.map((t) => [t]));
|
|
12191
|
+
}
|
|
12192
|
+
/**
|
|
12193
|
+
* Extract feature values from a record
|
|
12194
|
+
* @private
|
|
12195
|
+
* @param {Object} record - Data record
|
|
12196
|
+
* @returns {Array} Feature values
|
|
12197
|
+
*/
|
|
12198
|
+
_extractFeatures(record) {
|
|
12199
|
+
return this.config.features.map((feature) => {
|
|
12200
|
+
const value = record[feature];
|
|
12201
|
+
if (typeof value !== "number") {
|
|
12202
|
+
throw new DataValidationError(
|
|
12203
|
+
`Feature "${feature}" must be a number, got ${typeof value}`,
|
|
12204
|
+
{ model: this.config.name, feature, value, type: typeof value }
|
|
12205
|
+
);
|
|
12206
|
+
}
|
|
12207
|
+
return value;
|
|
12208
|
+
});
|
|
12209
|
+
}
|
|
12210
|
+
/**
|
|
12211
|
+
* Calculate normalization parameters (min-max scaling)
|
|
12212
|
+
* @private
|
|
12213
|
+
*/
|
|
12214
|
+
_calculateNormalizer(features, targets) {
|
|
12215
|
+
const numFeatures = features[0].length;
|
|
12216
|
+
for (let i = 0; i < numFeatures; i++) {
|
|
12217
|
+
const featureName = this.config.features[i];
|
|
12218
|
+
const values = features.map((f) => f[i]);
|
|
12219
|
+
this.normalizer.features[featureName] = {
|
|
12220
|
+
min: Math.min(...values),
|
|
12221
|
+
max: Math.max(...values)
|
|
12222
|
+
};
|
|
12223
|
+
}
|
|
12224
|
+
this.normalizer.target = {
|
|
12225
|
+
min: Math.min(...targets),
|
|
12226
|
+
max: Math.max(...targets)
|
|
12227
|
+
};
|
|
12228
|
+
}
|
|
12229
|
+
/**
|
|
12230
|
+
* Normalize features using min-max scaling
|
|
12231
|
+
* @private
|
|
12232
|
+
*/
|
|
12233
|
+
_normalizeFeatures(features) {
|
|
12234
|
+
return features.map((value, i) => {
|
|
12235
|
+
const featureName = this.config.features[i];
|
|
12236
|
+
const { min, max } = this.normalizer.features[featureName];
|
|
12237
|
+
if (max === min) return 0.5;
|
|
12238
|
+
return (value - min) / (max - min);
|
|
12239
|
+
});
|
|
12240
|
+
}
|
|
12241
|
+
/**
|
|
12242
|
+
* Normalize target value
|
|
12243
|
+
* @private
|
|
12244
|
+
*/
|
|
12245
|
+
_normalizeTarget(target) {
|
|
12246
|
+
const { min, max } = this.normalizer.target;
|
|
12247
|
+
if (max === min) return 0.5;
|
|
12248
|
+
return (target - min) / (max - min);
|
|
12249
|
+
}
|
|
12250
|
+
/**
|
|
12251
|
+
* Denormalize prediction
|
|
12252
|
+
* @private
|
|
12253
|
+
*/
|
|
12254
|
+
_denormalizePrediction(normalizedValue) {
|
|
12255
|
+
const { min, max } = this.normalizer.target;
|
|
12256
|
+
return normalizedValue * (max - min) + min;
|
|
12257
|
+
}
|
|
12258
|
+
/**
|
|
12259
|
+
* Calculate confidence score (can be overridden)
|
|
12260
|
+
* @protected
|
|
12261
|
+
*/
|
|
12262
|
+
_calculateConfidence(value) {
|
|
12263
|
+
const distanceFrom05 = Math.abs(value - 0.5);
|
|
12264
|
+
return Math.min(0.5 + distanceFrom05, 1);
|
|
12265
|
+
}
|
|
12266
|
+
/**
|
|
12267
|
+
* Validate input data
|
|
12268
|
+
* @private
|
|
12269
|
+
*/
|
|
12270
|
+
_validateInput(input) {
|
|
12271
|
+
const missingFeatures = this.config.features.filter((f) => !(f in input));
|
|
12272
|
+
if (missingFeatures.length > 0) {
|
|
12273
|
+
throw new DataValidationError(
|
|
12274
|
+
`Missing features: ${missingFeatures.join(", ")}`,
|
|
12275
|
+
{ model: this.config.name, missingFeatures, input }
|
|
12276
|
+
);
|
|
12277
|
+
}
|
|
12278
|
+
}
|
|
12279
|
+
/**
|
|
12280
|
+
* Export model to JSON (for persistence)
|
|
12281
|
+
* @returns {Object} Serialized model
|
|
12282
|
+
*/
|
|
12283
|
+
async export() {
|
|
12284
|
+
if (!this.model) {
|
|
12285
|
+
return null;
|
|
12286
|
+
}
|
|
12287
|
+
const modelJSON = await this.model.toJSON();
|
|
12288
|
+
return {
|
|
12289
|
+
config: this.config,
|
|
12290
|
+
normalizer: this.normalizer,
|
|
12291
|
+
stats: this.stats,
|
|
12292
|
+
isTrained: this.isTrained,
|
|
12293
|
+
model: modelJSON
|
|
12294
|
+
};
|
|
12295
|
+
}
|
|
12296
|
+
/**
|
|
12297
|
+
* Import model from JSON
|
|
12298
|
+
* @param {Object} data - Serialized model data
|
|
12299
|
+
*/
|
|
12300
|
+
async import(data) {
|
|
12301
|
+
this.config = data.config;
|
|
12302
|
+
this.normalizer = data.normalizer;
|
|
12303
|
+
this.stats = data.stats;
|
|
12304
|
+
this.isTrained = data.isTrained;
|
|
12305
|
+
if (data.model) {
|
|
12306
|
+
this.buildModel();
|
|
12307
|
+
}
|
|
12308
|
+
}
|
|
12309
|
+
/**
|
|
12310
|
+
* Dispose model and free memory
|
|
12311
|
+
*/
|
|
12312
|
+
dispose() {
|
|
12313
|
+
if (this.model) {
|
|
12314
|
+
this.model.dispose();
|
|
12315
|
+
this.model = null;
|
|
12316
|
+
}
|
|
12317
|
+
this.isTrained = false;
|
|
12318
|
+
}
|
|
12319
|
+
/**
|
|
12320
|
+
* Get model statistics
|
|
12321
|
+
*/
|
|
12322
|
+
getStats() {
|
|
12323
|
+
return {
|
|
12324
|
+
...this.stats,
|
|
12325
|
+
isTrained: this.isTrained,
|
|
12326
|
+
config: this.config
|
|
12327
|
+
};
|
|
12328
|
+
}
|
|
12329
|
+
}
|
|
12330
|
+
|
|
12331
|
+
class RegressionModel extends BaseModel {
|
|
12332
|
+
constructor(config = {}) {
|
|
12333
|
+
super(config);
|
|
12334
|
+
this.config.modelConfig = {
|
|
12335
|
+
...this.config.modelConfig,
|
|
12336
|
+
polynomial: config.modelConfig?.polynomial || 1,
|
|
12337
|
+
// Degree (1 = linear, 2+ = polynomial)
|
|
12338
|
+
units: config.modelConfig?.units || 64,
|
|
12339
|
+
// Hidden layer units for polynomial regression
|
|
12340
|
+
activation: config.modelConfig?.activation || "relu"
|
|
12341
|
+
};
|
|
12342
|
+
if (this.config.modelConfig.polynomial < 1 || this.config.modelConfig.polynomial > 5) {
|
|
12343
|
+
throw new ModelConfigError(
|
|
12344
|
+
"Polynomial degree must be between 1 and 5",
|
|
12345
|
+
{ model: this.config.name, polynomial: this.config.modelConfig.polynomial }
|
|
12346
|
+
);
|
|
12347
|
+
}
|
|
12348
|
+
}
|
|
12349
|
+
/**
|
|
12350
|
+
* Build regression model architecture
|
|
12351
|
+
*/
|
|
12352
|
+
buildModel() {
|
|
12353
|
+
const numFeatures = this.config.features.length;
|
|
12354
|
+
const polynomial = this.config.modelConfig.polynomial;
|
|
12355
|
+
this.model = this.tf.sequential();
|
|
12356
|
+
if (polynomial === 1) {
|
|
12357
|
+
this.model.add(this.tf.layers.dense({
|
|
12358
|
+
inputShape: [numFeatures],
|
|
12359
|
+
units: 1,
|
|
12360
|
+
useBias: true
|
|
12361
|
+
}));
|
|
12362
|
+
} else {
|
|
12363
|
+
this.model.add(this.tf.layers.dense({
|
|
12364
|
+
inputShape: [numFeatures],
|
|
12365
|
+
units: this.config.modelConfig.units,
|
|
12366
|
+
activation: this.config.modelConfig.activation,
|
|
12367
|
+
useBias: true
|
|
12368
|
+
}));
|
|
12369
|
+
if (polynomial >= 3) {
|
|
12370
|
+
this.model.add(this.tf.layers.dense({
|
|
12371
|
+
units: Math.floor(this.config.modelConfig.units / 2),
|
|
12372
|
+
activation: this.config.modelConfig.activation
|
|
12373
|
+
}));
|
|
12374
|
+
}
|
|
12375
|
+
this.model.add(this.tf.layers.dense({
|
|
12376
|
+
units: 1
|
|
12377
|
+
}));
|
|
12378
|
+
}
|
|
12379
|
+
this.model.compile({
|
|
12380
|
+
optimizer: this.tf.train.adam(this.config.modelConfig.learningRate),
|
|
12381
|
+
loss: "meanSquaredError",
|
|
12382
|
+
metrics: ["mse", "mae"]
|
|
12383
|
+
});
|
|
12384
|
+
if (this.config.verbose) {
|
|
12385
|
+
console.log(`[MLPlugin] ${this.config.name} - Built regression model (polynomial degree: ${polynomial})`);
|
|
12386
|
+
this.model.summary();
|
|
12387
|
+
}
|
|
12388
|
+
}
|
|
12389
|
+
/**
|
|
12390
|
+
* Override confidence calculation for regression
|
|
12391
|
+
* Uses prediction variance/uncertainty as confidence
|
|
12392
|
+
* @protected
|
|
12393
|
+
*/
|
|
12394
|
+
_calculateConfidence(value) {
|
|
12395
|
+
if (value >= 0 && value <= 1) {
|
|
12396
|
+
return 0.9 + Math.random() * 0.1;
|
|
12397
|
+
}
|
|
12398
|
+
const distance = Math.abs(value < 0 ? value : value - 1);
|
|
12399
|
+
return Math.max(0.5, 1 - distance);
|
|
12400
|
+
}
|
|
12401
|
+
/**
|
|
12402
|
+
* Get R² score (coefficient of determination)
|
|
12403
|
+
* Measures how well the model explains the variance in the data
|
|
12404
|
+
* @param {Array} data - Test data
|
|
12405
|
+
* @returns {number} R² score (0-1, higher is better)
|
|
12406
|
+
*/
|
|
12407
|
+
async calculateR2Score(data) {
|
|
12408
|
+
if (!this.isTrained) {
|
|
12409
|
+
throw new ModelNotTrainedError(`Model "${this.config.name}" is not trained yet`, {
|
|
12410
|
+
model: this.config.name
|
|
12411
|
+
});
|
|
12412
|
+
}
|
|
12413
|
+
const predictions = [];
|
|
12414
|
+
const actuals = [];
|
|
12415
|
+
for (const record of data) {
|
|
12416
|
+
const { prediction } = await this.predict(record);
|
|
12417
|
+
predictions.push(prediction);
|
|
12418
|
+
actuals.push(record[this.config.target]);
|
|
12419
|
+
}
|
|
12420
|
+
const meanActual = actuals.reduce((sum, val) => sum + val, 0) / actuals.length;
|
|
12421
|
+
const tss = actuals.reduce((sum, actual) => {
|
|
12422
|
+
return sum + Math.pow(actual - meanActual, 2);
|
|
12423
|
+
}, 0);
|
|
12424
|
+
const rss = predictions.reduce((sum, pred, i) => {
|
|
12425
|
+
return sum + Math.pow(actuals[i] - pred, 2);
|
|
12426
|
+
}, 0);
|
|
12427
|
+
const r2 = 1 - rss / tss;
|
|
12428
|
+
return r2;
|
|
12429
|
+
}
|
|
12430
|
+
/**
|
|
12431
|
+
* Export model with regression-specific data
|
|
12432
|
+
*/
|
|
12433
|
+
async export() {
|
|
12434
|
+
const baseExport = await super.export();
|
|
12435
|
+
return {
|
|
12436
|
+
...baseExport,
|
|
12437
|
+
type: "regression",
|
|
12438
|
+
polynomial: this.config.modelConfig.polynomial
|
|
12439
|
+
};
|
|
12440
|
+
}
|
|
12441
|
+
}
|
|
12442
|
+
|
|
12443
|
+
class ClassificationModel extends BaseModel {
|
|
12444
|
+
constructor(config = {}) {
|
|
12445
|
+
super(config);
|
|
12446
|
+
this.config.modelConfig = {
|
|
12447
|
+
...this.config.modelConfig,
|
|
12448
|
+
units: config.modelConfig?.units || 64,
|
|
12449
|
+
// Hidden layer units
|
|
12450
|
+
activation: config.modelConfig?.activation || "relu",
|
|
12451
|
+
dropout: config.modelConfig?.dropout || 0.2
|
|
12452
|
+
// Dropout rate for regularization
|
|
12453
|
+
};
|
|
12454
|
+
this.classes = [];
|
|
12455
|
+
this.classToIndex = {};
|
|
12456
|
+
this.indexToClass = {};
|
|
12457
|
+
}
|
|
12458
|
+
/**
|
|
12459
|
+
* Build classification model architecture
|
|
12460
|
+
*/
|
|
12461
|
+
buildModel() {
|
|
12462
|
+
const numFeatures = this.config.features.length;
|
|
12463
|
+
const numClasses = this.classes.length;
|
|
12464
|
+
if (numClasses < 2) {
|
|
12465
|
+
throw new ModelConfigError(
|
|
12466
|
+
"Classification requires at least 2 classes",
|
|
12467
|
+
{ model: this.config.name, numClasses }
|
|
12468
|
+
);
|
|
12469
|
+
}
|
|
12470
|
+
this.model = this.tf.sequential();
|
|
12471
|
+
this.model.add(this.tf.layers.dense({
|
|
12472
|
+
inputShape: [numFeatures],
|
|
12473
|
+
units: this.config.modelConfig.units,
|
|
12474
|
+
activation: this.config.modelConfig.activation,
|
|
12475
|
+
useBias: true
|
|
12476
|
+
}));
|
|
12477
|
+
if (this.config.modelConfig.dropout > 0) {
|
|
12478
|
+
this.model.add(this.tf.layers.dropout({
|
|
12479
|
+
rate: this.config.modelConfig.dropout
|
|
12480
|
+
}));
|
|
12481
|
+
}
|
|
12482
|
+
this.model.add(this.tf.layers.dense({
|
|
12483
|
+
units: Math.floor(this.config.modelConfig.units / 2),
|
|
12484
|
+
activation: this.config.modelConfig.activation
|
|
12485
|
+
}));
|
|
12486
|
+
const isBinary = numClasses === 2;
|
|
12487
|
+
this.model.add(this.tf.layers.dense({
|
|
12488
|
+
units: isBinary ? 1 : numClasses,
|
|
12489
|
+
activation: isBinary ? "sigmoid" : "softmax"
|
|
12490
|
+
}));
|
|
12491
|
+
this.model.compile({
|
|
12492
|
+
optimizer: this.tf.train.adam(this.config.modelConfig.learningRate),
|
|
12493
|
+
loss: isBinary ? "binaryCrossentropy" : "categoricalCrossentropy",
|
|
12494
|
+
metrics: ["accuracy"]
|
|
12495
|
+
});
|
|
12496
|
+
if (this.config.verbose) {
|
|
12497
|
+
console.log(`[MLPlugin] ${this.config.name} - Built classification model (${numClasses} classes, ${isBinary ? "binary" : "multi-class"})`);
|
|
12498
|
+
this.model.summary();
|
|
12499
|
+
}
|
|
12500
|
+
}
|
|
12501
|
+
/**
|
|
12502
|
+
* Prepare training data (override to handle class labels)
|
|
12503
|
+
* @private
|
|
12504
|
+
*/
|
|
12505
|
+
_prepareData(data) {
|
|
12506
|
+
const features = [];
|
|
12507
|
+
const targets = [];
|
|
12508
|
+
const uniqueClasses = [...new Set(data.map((r) => r[this.config.target]))];
|
|
12509
|
+
this.classes = uniqueClasses.sort();
|
|
12510
|
+
this.classes.forEach((cls, idx) => {
|
|
12511
|
+
this.classToIndex[cls] = idx;
|
|
12512
|
+
this.indexToClass[idx] = cls;
|
|
12513
|
+
});
|
|
12514
|
+
if (this.config.verbose) {
|
|
12515
|
+
console.log(`[MLPlugin] ${this.config.name} - Detected ${this.classes.length} classes:`, this.classes);
|
|
12516
|
+
}
|
|
12517
|
+
for (const record of data) {
|
|
12518
|
+
const missingFeatures = this.config.features.filter((f) => !(f in record));
|
|
12519
|
+
if (missingFeatures.length > 0) {
|
|
12520
|
+
throw new DataValidationError(
|
|
12521
|
+
`Missing features in training data: ${missingFeatures.join(", ")}`,
|
|
12522
|
+
{ model: this.config.name, missingFeatures, record }
|
|
12523
|
+
);
|
|
12524
|
+
}
|
|
12525
|
+
if (!(this.config.target in record)) {
|
|
12526
|
+
throw new DataValidationError(
|
|
12527
|
+
`Missing target "${this.config.target}" in training data`,
|
|
12528
|
+
{ model: this.config.name, target: this.config.target, record }
|
|
12529
|
+
);
|
|
12530
|
+
}
|
|
12531
|
+
const featureValues = this._extractFeatures(record);
|
|
12532
|
+
features.push(featureValues);
|
|
12533
|
+
const targetClass = record[this.config.target];
|
|
12534
|
+
if (!(targetClass in this.classToIndex)) {
|
|
12535
|
+
throw new DataValidationError(
|
|
12536
|
+
`Unknown class "${targetClass}" in training data`,
|
|
12537
|
+
{ model: this.config.name, targetClass, knownClasses: this.classes }
|
|
12538
|
+
);
|
|
12539
|
+
}
|
|
12540
|
+
targets.push(this.classToIndex[targetClass]);
|
|
12541
|
+
}
|
|
12542
|
+
this._calculateNormalizer(features, targets);
|
|
12543
|
+
const normalizedFeatures = features.map((f) => this._normalizeFeatures(f));
|
|
12544
|
+
return {
|
|
12545
|
+
xs: this.tf.tensor2d(normalizedFeatures),
|
|
12546
|
+
ys: this._prepareTargetTensor(targets)
|
|
12547
|
+
};
|
|
12548
|
+
}
|
|
12549
|
+
/**
|
|
12550
|
+
* Prepare target tensor for classification (one-hot encoding or binary)
|
|
12551
|
+
* @protected
|
|
12552
|
+
*/
|
|
12553
|
+
_prepareTargetTensor(targets) {
|
|
12554
|
+
const isBinary = this.classes.length === 2;
|
|
12555
|
+
if (isBinary) {
|
|
12556
|
+
return this.tf.tensor2d(targets.map((t) => [t]));
|
|
12557
|
+
} else {
|
|
12558
|
+
return this.tf.oneHot(targets, this.classes.length);
|
|
12559
|
+
}
|
|
12560
|
+
}
|
|
12561
|
+
/**
|
|
12562
|
+
* Calculate normalization parameters (skip target normalization for classification)
|
|
12563
|
+
* @private
|
|
12564
|
+
*/
|
|
12565
|
+
_calculateNormalizer(features, targets) {
|
|
12566
|
+
const numFeatures = features[0].length;
|
|
12567
|
+
for (let i = 0; i < numFeatures; i++) {
|
|
12568
|
+
const featureName = this.config.features[i];
|
|
12569
|
+
const values = features.map((f) => f[i]);
|
|
12570
|
+
this.normalizer.features[featureName] = {
|
|
12571
|
+
min: Math.min(...values),
|
|
12572
|
+
max: Math.max(...values)
|
|
12573
|
+
};
|
|
12574
|
+
}
|
|
12575
|
+
this.normalizer.target = { min: 0, max: 1 };
|
|
12576
|
+
}
|
|
12577
|
+
/**
|
|
12578
|
+
* Make a prediction (override to return class label)
|
|
12579
|
+
*/
|
|
12580
|
+
async predict(input) {
|
|
12581
|
+
if (!this.isTrained) {
|
|
12582
|
+
throw new ModelNotTrainedError(`Model "${this.config.name}" is not trained yet`, {
|
|
12583
|
+
model: this.config.name
|
|
12584
|
+
});
|
|
12585
|
+
}
|
|
12586
|
+
try {
|
|
12587
|
+
this._validateInput(input);
|
|
12588
|
+
const features = this._extractFeatures(input);
|
|
12589
|
+
const normalizedFeatures = this._normalizeFeatures(features);
|
|
12590
|
+
const inputTensor = this.tf.tensor2d([normalizedFeatures]);
|
|
12591
|
+
const predictionTensor = this.model.predict(inputTensor);
|
|
12592
|
+
const predictionArray = await predictionTensor.data();
|
|
12593
|
+
inputTensor.dispose();
|
|
12594
|
+
predictionTensor.dispose();
|
|
12595
|
+
const isBinary = this.classes.length === 2;
|
|
12596
|
+
let predictedClassIndex;
|
|
12597
|
+
let confidence;
|
|
12598
|
+
if (isBinary) {
|
|
12599
|
+
confidence = predictionArray[0];
|
|
12600
|
+
predictedClassIndex = confidence >= 0.5 ? 1 : 0;
|
|
12601
|
+
} else {
|
|
12602
|
+
predictedClassIndex = predictionArray.indexOf(Math.max(...predictionArray));
|
|
12603
|
+
confidence = predictionArray[predictedClassIndex];
|
|
12604
|
+
}
|
|
12605
|
+
const predictedClass = this.indexToClass[predictedClassIndex];
|
|
12606
|
+
this.stats.predictions++;
|
|
12607
|
+
return {
|
|
12608
|
+
prediction: predictedClass,
|
|
12609
|
+
confidence,
|
|
12610
|
+
probabilities: isBinary ? {
|
|
12611
|
+
[this.classes[0]]: 1 - predictionArray[0],
|
|
12612
|
+
[this.classes[1]]: predictionArray[0]
|
|
12613
|
+
} : Object.fromEntries(
|
|
12614
|
+
this.classes.map((cls, idx) => [cls, predictionArray[idx]])
|
|
12615
|
+
)
|
|
12616
|
+
};
|
|
12617
|
+
} catch (error) {
|
|
12618
|
+
this.stats.errors++;
|
|
12619
|
+
if (error instanceof ModelNotTrainedError || error instanceof DataValidationError) {
|
|
12620
|
+
throw error;
|
|
12621
|
+
}
|
|
12622
|
+
throw new PredictionError(`Prediction failed: ${error.message}`, {
|
|
12623
|
+
model: this.config.name,
|
|
12624
|
+
input,
|
|
12625
|
+
originalError: error.message
|
|
12626
|
+
});
|
|
12627
|
+
}
|
|
12628
|
+
}
|
|
12629
|
+
/**
|
|
12630
|
+
* Calculate confusion matrix
|
|
12631
|
+
* @param {Array} data - Test data
|
|
12632
|
+
* @returns {Object} Confusion matrix and metrics
|
|
12633
|
+
*/
|
|
12634
|
+
async calculateConfusionMatrix(data) {
|
|
12635
|
+
if (!this.isTrained) {
|
|
12636
|
+
throw new ModelNotTrainedError(`Model "${this.config.name}" is not trained yet`, {
|
|
12637
|
+
model: this.config.name
|
|
12638
|
+
});
|
|
12639
|
+
}
|
|
12640
|
+
const matrix = {};
|
|
12641
|
+
this.classes.length;
|
|
12642
|
+
for (const actualClass of this.classes) {
|
|
12643
|
+
matrix[actualClass] = {};
|
|
12644
|
+
for (const predictedClass of this.classes) {
|
|
12645
|
+
matrix[actualClass][predictedClass] = 0;
|
|
12646
|
+
}
|
|
12647
|
+
}
|
|
12648
|
+
for (const record of data) {
|
|
12649
|
+
const { prediction } = await this.predict(record);
|
|
12650
|
+
const actual = record[this.config.target];
|
|
12651
|
+
matrix[actual][prediction]++;
|
|
12652
|
+
}
|
|
12653
|
+
let totalCorrect = 0;
|
|
12654
|
+
let total = 0;
|
|
12655
|
+
for (const cls of this.classes) {
|
|
12656
|
+
totalCorrect += matrix[cls][cls];
|
|
12657
|
+
total += Object.values(matrix[cls]).reduce((sum, val) => sum + val, 0);
|
|
12658
|
+
}
|
|
12659
|
+
const accuracy = total > 0 ? totalCorrect / total : 0;
|
|
12660
|
+
return {
|
|
12661
|
+
matrix,
|
|
12662
|
+
accuracy,
|
|
12663
|
+
total,
|
|
12664
|
+
correct: totalCorrect
|
|
12665
|
+
};
|
|
12666
|
+
}
|
|
12667
|
+
/**
|
|
12668
|
+
* Export model with classification-specific data
|
|
12669
|
+
*/
|
|
12670
|
+
async export() {
|
|
12671
|
+
const baseExport = await super.export();
|
|
12672
|
+
return {
|
|
12673
|
+
...baseExport,
|
|
12674
|
+
type: "classification",
|
|
12675
|
+
classes: this.classes,
|
|
12676
|
+
classToIndex: this.classToIndex,
|
|
12677
|
+
indexToClass: this.indexToClass
|
|
12678
|
+
};
|
|
12679
|
+
}
|
|
12680
|
+
/**
|
|
12681
|
+
* Import model (override to restore class mappings)
|
|
12682
|
+
*/
|
|
12683
|
+
async import(data) {
|
|
12684
|
+
await super.import(data);
|
|
12685
|
+
this.classes = data.classes || [];
|
|
12686
|
+
this.classToIndex = data.classToIndex || {};
|
|
12687
|
+
this.indexToClass = data.indexToClass || {};
|
|
12688
|
+
}
|
|
12689
|
+
}
|
|
12690
|
+
|
|
12691
|
+
class TimeSeriesModel extends BaseModel {
|
|
12692
|
+
constructor(config = {}) {
|
|
12693
|
+
super(config);
|
|
12694
|
+
this.config.modelConfig = {
|
|
12695
|
+
...this.config.modelConfig,
|
|
12696
|
+
lookback: config.modelConfig?.lookback || 10,
|
|
12697
|
+
// Number of past timesteps to use
|
|
12698
|
+
lstmUnits: config.modelConfig?.lstmUnits || 50,
|
|
12699
|
+
// LSTM layer units
|
|
12700
|
+
denseUnits: config.modelConfig?.denseUnits || 25,
|
|
12701
|
+
// Dense layer units
|
|
12702
|
+
dropout: config.modelConfig?.dropout || 0.2,
|
|
12703
|
+
recurrentDropout: config.modelConfig?.recurrentDropout || 0.2
|
|
12704
|
+
};
|
|
12705
|
+
if (this.config.modelConfig.lookback < 2) {
|
|
12706
|
+
throw new ModelConfigError(
|
|
12707
|
+
"Lookback window must be at least 2",
|
|
12708
|
+
{ model: this.config.name, lookback: this.config.modelConfig.lookback }
|
|
12709
|
+
);
|
|
12710
|
+
}
|
|
12711
|
+
}
|
|
12712
|
+
/**
|
|
12713
|
+
* Build LSTM model architecture for time series
|
|
12714
|
+
*/
|
|
12715
|
+
buildModel() {
|
|
12716
|
+
const numFeatures = this.config.features.length + 1;
|
|
12717
|
+
const lookback = this.config.modelConfig.lookback;
|
|
12718
|
+
this.model = this.tf.sequential();
|
|
12719
|
+
this.model.add(this.tf.layers.lstm({
|
|
12720
|
+
inputShape: [lookback, numFeatures],
|
|
12721
|
+
units: this.config.modelConfig.lstmUnits,
|
|
12722
|
+
returnSequences: false,
|
|
12723
|
+
dropout: this.config.modelConfig.dropout,
|
|
12724
|
+
recurrentDropout: this.config.modelConfig.recurrentDropout
|
|
12725
|
+
}));
|
|
12726
|
+
this.model.add(this.tf.layers.dense({
|
|
12727
|
+
units: this.config.modelConfig.denseUnits,
|
|
12728
|
+
activation: "relu"
|
|
12729
|
+
}));
|
|
12730
|
+
if (this.config.modelConfig.dropout > 0) {
|
|
12731
|
+
this.model.add(this.tf.layers.dropout({
|
|
12732
|
+
rate: this.config.modelConfig.dropout
|
|
12733
|
+
}));
|
|
12734
|
+
}
|
|
12735
|
+
this.model.add(this.tf.layers.dense({
|
|
12736
|
+
units: 1
|
|
12737
|
+
}));
|
|
12738
|
+
this.model.compile({
|
|
12739
|
+
optimizer: this.tf.train.adam(this.config.modelConfig.learningRate),
|
|
12740
|
+
loss: "meanSquaredError",
|
|
12741
|
+
metrics: ["mse", "mae"]
|
|
12742
|
+
});
|
|
12743
|
+
if (this.config.verbose) {
|
|
12744
|
+
console.log(`[MLPlugin] ${this.config.name} - Built LSTM time series model (lookback: ${lookback})`);
|
|
12745
|
+
this.model.summary();
|
|
12746
|
+
}
|
|
12747
|
+
}
|
|
12748
|
+
/**
|
|
12749
|
+
* Prepare time series data with sliding window
|
|
12750
|
+
* @private
|
|
12751
|
+
*/
|
|
12752
|
+
_prepareData(data) {
|
|
12753
|
+
const lookback = this.config.modelConfig.lookback;
|
|
12754
|
+
if (data.length < lookback + 1) {
|
|
12755
|
+
throw new InsufficientDataError(
|
|
12756
|
+
`Insufficient time series data: ${data.length} samples (minimum: ${lookback + 1})`,
|
|
12757
|
+
{ model: this.config.name, samples: data.length, minimum: lookback + 1 }
|
|
12758
|
+
);
|
|
12759
|
+
}
|
|
12760
|
+
const sequences = [];
|
|
12761
|
+
const targets = [];
|
|
12762
|
+
const allValues = [];
|
|
12763
|
+
for (const record of data) {
|
|
12764
|
+
const features = this._extractFeatures(record);
|
|
12765
|
+
const target = record[this.config.target];
|
|
12766
|
+
allValues.push([...features, target]);
|
|
12767
|
+
}
|
|
12768
|
+
this._calculateTimeSeriesNormalizer(allValues);
|
|
12769
|
+
for (let i = 0; i <= data.length - lookback - 1; i++) {
|
|
12770
|
+
const sequence = [];
|
|
12771
|
+
for (let j = 0; j < lookback; j++) {
|
|
12772
|
+
const record = data[i + j];
|
|
12773
|
+
const features = this._extractFeatures(record);
|
|
12774
|
+
const target = record[this.config.target];
|
|
12775
|
+
const combined = [...features, target];
|
|
12776
|
+
const normalized = this._normalizeSequenceStep(combined);
|
|
12777
|
+
sequence.push(normalized);
|
|
12778
|
+
}
|
|
12779
|
+
const nextRecord = data[i + lookback];
|
|
12780
|
+
const nextTarget = nextRecord[this.config.target];
|
|
12781
|
+
sequences.push(sequence);
|
|
12782
|
+
targets.push(this._normalizeTarget(nextTarget));
|
|
12783
|
+
}
|
|
12784
|
+
return {
|
|
12785
|
+
xs: this.tf.tensor3d(sequences),
|
|
12786
|
+
// [samples, lookback, features]
|
|
12787
|
+
ys: this.tf.tensor2d(targets.map((t) => [t]))
|
|
12788
|
+
// [samples, 1]
|
|
12789
|
+
};
|
|
12790
|
+
}
|
|
12791
|
+
/**
|
|
12792
|
+
* Calculate normalization for time series
|
|
12793
|
+
* @private
|
|
12794
|
+
*/
|
|
12795
|
+
_calculateTimeSeriesNormalizer(allValues) {
|
|
12796
|
+
const numFeatures = allValues[0].length;
|
|
12797
|
+
for (let i = 0; i < numFeatures; i++) {
|
|
12798
|
+
const values = allValues.map((v) => v[i]);
|
|
12799
|
+
const min = Math.min(...values);
|
|
12800
|
+
const max = Math.max(...values);
|
|
12801
|
+
if (i < this.config.features.length) {
|
|
12802
|
+
const featureName = this.config.features[i];
|
|
12803
|
+
this.normalizer.features[featureName] = { min, max };
|
|
12804
|
+
} else {
|
|
12805
|
+
this.normalizer.target = { min, max };
|
|
12806
|
+
}
|
|
12807
|
+
}
|
|
12808
|
+
}
|
|
12809
|
+
/**
|
|
12810
|
+
* Normalize a sequence step (features + target)
|
|
12811
|
+
* @private
|
|
12812
|
+
*/
|
|
12813
|
+
_normalizeSequenceStep(values) {
|
|
12814
|
+
return values.map((value, i) => {
|
|
12815
|
+
let min, max;
|
|
12816
|
+
if (i < this.config.features.length) {
|
|
12817
|
+
const featureName = this.config.features[i];
|
|
12818
|
+
({ min, max } = this.normalizer.features[featureName]);
|
|
12819
|
+
} else {
|
|
12820
|
+
({ min, max } = this.normalizer.target);
|
|
12821
|
+
}
|
|
12822
|
+
if (max === min) return 0.5;
|
|
12823
|
+
return (value - min) / (max - min);
|
|
12824
|
+
});
|
|
12825
|
+
}
|
|
12826
|
+
/**
|
|
12827
|
+
* Predict next value in time series
|
|
12828
|
+
* @param {Array} sequence - Array of recent records (length = lookback)
|
|
12829
|
+
* @returns {Object} Prediction result
|
|
12830
|
+
*/
|
|
12831
|
+
async predict(sequence) {
|
|
12832
|
+
if (!this.isTrained) {
|
|
12833
|
+
throw new ModelNotTrainedError(`Model "${this.config.name}" is not trained yet`, {
|
|
12834
|
+
model: this.config.name
|
|
12835
|
+
});
|
|
12836
|
+
}
|
|
12837
|
+
try {
|
|
12838
|
+
if (!Array.isArray(sequence)) {
|
|
12839
|
+
throw new DataValidationError(
|
|
12840
|
+
"Time series prediction requires an array of recent records",
|
|
12841
|
+
{ model: this.config.name, input: typeof sequence }
|
|
12842
|
+
);
|
|
12843
|
+
}
|
|
12844
|
+
if (sequence.length !== this.config.modelConfig.lookback) {
|
|
12845
|
+
throw new DataValidationError(
|
|
12846
|
+
`Time series sequence must have exactly ${this.config.modelConfig.lookback} timesteps, got ${sequence.length}`,
|
|
12847
|
+
{ model: this.config.name, expected: this.config.modelConfig.lookback, got: sequence.length }
|
|
12848
|
+
);
|
|
12849
|
+
}
|
|
12850
|
+
const normalizedSequence = [];
|
|
12851
|
+
for (const record of sequence) {
|
|
12852
|
+
this._validateInput(record);
|
|
12853
|
+
const features = this._extractFeatures(record);
|
|
12854
|
+
const target = record[this.config.target];
|
|
12855
|
+
const combined = [...features, target];
|
|
12856
|
+
normalizedSequence.push(this._normalizeSequenceStep(combined));
|
|
12857
|
+
}
|
|
12858
|
+
const inputTensor = this.tf.tensor3d([normalizedSequence]);
|
|
12859
|
+
const predictionTensor = this.model.predict(inputTensor);
|
|
12860
|
+
const predictionArray = await predictionTensor.data();
|
|
12861
|
+
inputTensor.dispose();
|
|
12862
|
+
predictionTensor.dispose();
|
|
12863
|
+
const prediction = this._denormalizePrediction(predictionArray[0]);
|
|
12864
|
+
this.stats.predictions++;
|
|
12865
|
+
return {
|
|
12866
|
+
prediction,
|
|
12867
|
+
confidence: this._calculateConfidence(predictionArray[0])
|
|
12868
|
+
};
|
|
12869
|
+
} catch (error) {
|
|
12870
|
+
this.stats.errors++;
|
|
12871
|
+
if (error instanceof ModelNotTrainedError || error instanceof DataValidationError) {
|
|
12872
|
+
throw error;
|
|
12873
|
+
}
|
|
12874
|
+
throw new PredictionError(`Time series prediction failed: ${error.message}`, {
|
|
12875
|
+
model: this.config.name,
|
|
12876
|
+
originalError: error.message
|
|
12877
|
+
});
|
|
12878
|
+
}
|
|
12879
|
+
}
|
|
12880
|
+
/**
|
|
12881
|
+
* Predict multiple future timesteps
|
|
12882
|
+
* @param {Array} initialSequence - Initial sequence of records
|
|
12883
|
+
* @param {number} steps - Number of steps to predict ahead
|
|
12884
|
+
* @returns {Array} Array of predictions
|
|
12885
|
+
*/
|
|
12886
|
+
async predictMultiStep(initialSequence, steps = 1) {
|
|
12887
|
+
if (!this.isTrained) {
|
|
12888
|
+
throw new ModelNotTrainedError(`Model "${this.config.name}" is not trained yet`, {
|
|
12889
|
+
model: this.config.name
|
|
12890
|
+
});
|
|
12891
|
+
}
|
|
12892
|
+
const predictions = [];
|
|
12893
|
+
let currentSequence = [...initialSequence];
|
|
12894
|
+
for (let i = 0; i < steps; i++) {
|
|
12895
|
+
const { prediction } = await this.predict(currentSequence);
|
|
12896
|
+
predictions.push(prediction);
|
|
12897
|
+
currentSequence.shift();
|
|
12898
|
+
const lastRecord = currentSequence[currentSequence.length - 1];
|
|
12899
|
+
const syntheticRecord = {
|
|
12900
|
+
...lastRecord,
|
|
12901
|
+
[this.config.target]: prediction
|
|
12902
|
+
};
|
|
12903
|
+
currentSequence.push(syntheticRecord);
|
|
12904
|
+
}
|
|
12905
|
+
return predictions;
|
|
12906
|
+
}
|
|
12907
|
+
/**
|
|
12908
|
+
* Calculate Mean Absolute Percentage Error (MAPE)
|
|
12909
|
+
* @param {Array} data - Test data (must be sequential)
|
|
12910
|
+
* @returns {number} MAPE (0-100, lower is better)
|
|
12911
|
+
*/
|
|
12912
|
+
async calculateMAPE(data) {
|
|
12913
|
+
if (!this.isTrained) {
|
|
12914
|
+
throw new ModelNotTrainedError(`Model "${this.config.name}" is not trained yet`, {
|
|
12915
|
+
model: this.config.name
|
|
12916
|
+
});
|
|
12917
|
+
}
|
|
12918
|
+
const lookback = this.config.modelConfig.lookback;
|
|
12919
|
+
if (data.length < lookback + 1) {
|
|
12920
|
+
throw new InsufficientDataError(
|
|
12921
|
+
`Insufficient test data for MAPE calculation`,
|
|
12922
|
+
{ model: this.config.name, samples: data.length, minimum: lookback + 1 }
|
|
12923
|
+
);
|
|
12924
|
+
}
|
|
12925
|
+
let totalPercentageError = 0;
|
|
12926
|
+
let count = 0;
|
|
12927
|
+
for (let i = lookback; i < data.length; i++) {
|
|
12928
|
+
const sequence = data.slice(i - lookback, i);
|
|
12929
|
+
const { prediction } = await this.predict(sequence);
|
|
12930
|
+
const actual = data[i][this.config.target];
|
|
12931
|
+
if (actual !== 0) {
|
|
12932
|
+
const percentageError = Math.abs((actual - prediction) / actual) * 100;
|
|
12933
|
+
totalPercentageError += percentageError;
|
|
12934
|
+
count++;
|
|
12935
|
+
}
|
|
12936
|
+
}
|
|
12937
|
+
return count > 0 ? totalPercentageError / count : 0;
|
|
12938
|
+
}
|
|
12939
|
+
/**
|
|
12940
|
+
* Export model with time series-specific data
|
|
12941
|
+
*/
|
|
12942
|
+
async export() {
|
|
12943
|
+
const baseExport = await super.export();
|
|
12944
|
+
return {
|
|
12945
|
+
...baseExport,
|
|
12946
|
+
type: "timeseries",
|
|
12947
|
+
lookback: this.config.modelConfig.lookback
|
|
12948
|
+
};
|
|
12949
|
+
}
|
|
12950
|
+
}
|
|
12951
|
+
|
|
12952
|
+
class NeuralNetworkModel extends BaseModel {
|
|
12953
|
+
constructor(config = {}) {
|
|
12954
|
+
super(config);
|
|
12955
|
+
this.config.modelConfig = {
|
|
12956
|
+
...this.config.modelConfig,
|
|
12957
|
+
layers: config.modelConfig?.layers || [
|
|
12958
|
+
{ units: 64, activation: "relu", dropout: 0.2 },
|
|
12959
|
+
{ units: 32, activation: "relu", dropout: 0.1 }
|
|
12960
|
+
],
|
|
12961
|
+
// Array of hidden layer configurations
|
|
12962
|
+
outputActivation: config.modelConfig?.outputActivation || "linear",
|
|
12963
|
+
// Output layer activation
|
|
12964
|
+
outputUnits: config.modelConfig?.outputUnits || 1,
|
|
12965
|
+
// Number of output units
|
|
12966
|
+
loss: config.modelConfig?.loss || "meanSquaredError",
|
|
12967
|
+
// Loss function
|
|
12968
|
+
metrics: config.modelConfig?.metrics || ["mse", "mae"]
|
|
12969
|
+
// Metrics to track
|
|
12970
|
+
};
|
|
12971
|
+
this._validateLayersConfig();
|
|
12972
|
+
}
|
|
12973
|
+
/**
|
|
12974
|
+
* Validate layers configuration
|
|
12975
|
+
* @private
|
|
12976
|
+
*/
|
|
12977
|
+
_validateLayersConfig() {
|
|
12978
|
+
if (!Array.isArray(this.config.modelConfig.layers) || this.config.modelConfig.layers.length === 0) {
|
|
12979
|
+
throw new ModelConfigError(
|
|
12980
|
+
"Neural network must have at least one hidden layer",
|
|
12981
|
+
{ model: this.config.name, layers: this.config.modelConfig.layers }
|
|
12982
|
+
);
|
|
12983
|
+
}
|
|
12984
|
+
for (const [index, layer] of this.config.modelConfig.layers.entries()) {
|
|
12985
|
+
if (!layer.units || typeof layer.units !== "number" || layer.units < 1) {
|
|
12986
|
+
throw new ModelConfigError(
|
|
12987
|
+
`Layer ${index} must have a valid "units" property (positive number)`,
|
|
12988
|
+
{ model: this.config.name, layer, index }
|
|
12989
|
+
);
|
|
12990
|
+
}
|
|
12991
|
+
if (layer.activation && !this._isValidActivation(layer.activation)) {
|
|
12992
|
+
throw new ModelConfigError(
|
|
12993
|
+
`Layer ${index} has invalid activation function "${layer.activation}"`,
|
|
12994
|
+
{ model: this.config.name, layer, index, validActivations: ["relu", "sigmoid", "tanh", "softmax", "elu", "selu"] }
|
|
12995
|
+
);
|
|
12996
|
+
}
|
|
12997
|
+
}
|
|
12998
|
+
}
|
|
12999
|
+
/**
|
|
13000
|
+
* Check if activation function is valid
|
|
13001
|
+
* @private
|
|
13002
|
+
*/
|
|
13003
|
+
_isValidActivation(activation) {
|
|
13004
|
+
const validActivations = ["relu", "sigmoid", "tanh", "softmax", "elu", "selu", "linear"];
|
|
13005
|
+
return validActivations.includes(activation);
|
|
13006
|
+
}
|
|
13007
|
+
/**
|
|
13008
|
+
* Build custom neural network architecture
|
|
13009
|
+
*/
|
|
13010
|
+
buildModel() {
|
|
13011
|
+
const numFeatures = this.config.features.length;
|
|
13012
|
+
this.model = this.tf.sequential();
|
|
13013
|
+
for (const [index, layerConfig] of this.config.modelConfig.layers.entries()) {
|
|
13014
|
+
const isFirstLayer = index === 0;
|
|
13015
|
+
const layerOptions = {
|
|
13016
|
+
units: layerConfig.units,
|
|
13017
|
+
activation: layerConfig.activation || "relu",
|
|
13018
|
+
useBias: true
|
|
13019
|
+
};
|
|
13020
|
+
if (isFirstLayer) {
|
|
13021
|
+
layerOptions.inputShape = [numFeatures];
|
|
13022
|
+
}
|
|
13023
|
+
this.model.add(this.tf.layers.dense(layerOptions));
|
|
13024
|
+
if (layerConfig.dropout && layerConfig.dropout > 0) {
|
|
13025
|
+
this.model.add(this.tf.layers.dropout({
|
|
13026
|
+
rate: layerConfig.dropout
|
|
13027
|
+
}));
|
|
13028
|
+
}
|
|
13029
|
+
if (layerConfig.batchNormalization) {
|
|
13030
|
+
this.model.add(this.tf.layers.batchNormalization());
|
|
13031
|
+
}
|
|
13032
|
+
}
|
|
13033
|
+
this.model.add(this.tf.layers.dense({
|
|
13034
|
+
units: this.config.modelConfig.outputUnits,
|
|
13035
|
+
activation: this.config.modelConfig.outputActivation
|
|
13036
|
+
}));
|
|
13037
|
+
this.model.compile({
|
|
13038
|
+
optimizer: this.tf.train.adam(this.config.modelConfig.learningRate),
|
|
13039
|
+
loss: this.config.modelConfig.loss,
|
|
13040
|
+
metrics: this.config.modelConfig.metrics
|
|
13041
|
+
});
|
|
13042
|
+
if (this.config.verbose) {
|
|
13043
|
+
console.log(`[MLPlugin] ${this.config.name} - Built custom neural network:`);
|
|
13044
|
+
console.log(` - Hidden layers: ${this.config.modelConfig.layers.length}`);
|
|
13045
|
+
console.log(` - Total parameters:`, this._countParameters());
|
|
13046
|
+
this.model.summary();
|
|
13047
|
+
}
|
|
13048
|
+
}
|
|
13049
|
+
/**
|
|
13050
|
+
* Count total trainable parameters
|
|
13051
|
+
* @private
|
|
13052
|
+
*/
|
|
13053
|
+
_countParameters() {
|
|
13054
|
+
if (!this.model) return 0;
|
|
13055
|
+
let totalParams = 0;
|
|
13056
|
+
for (const layer of this.model.layers) {
|
|
13057
|
+
if (layer.countParams) {
|
|
13058
|
+
totalParams += layer.countParams();
|
|
13059
|
+
}
|
|
13060
|
+
}
|
|
13061
|
+
return totalParams;
|
|
13062
|
+
}
|
|
13063
|
+
/**
|
|
13064
|
+
* Add layer to model (before building)
|
|
13065
|
+
* @param {Object} layerConfig - Layer configuration
|
|
13066
|
+
*/
|
|
13067
|
+
addLayer(layerConfig) {
|
|
13068
|
+
if (this.model) {
|
|
13069
|
+
throw new ModelConfigError(
|
|
13070
|
+
"Cannot add layer after model is built. Use addLayer() before training.",
|
|
13071
|
+
{ model: this.config.name }
|
|
13072
|
+
);
|
|
13073
|
+
}
|
|
13074
|
+
this.config.modelConfig.layers.push(layerConfig);
|
|
13075
|
+
}
|
|
13076
|
+
/**
|
|
13077
|
+
* Set output configuration
|
|
13078
|
+
* @param {Object} outputConfig - Output layer configuration
|
|
13079
|
+
*/
|
|
13080
|
+
setOutput(outputConfig) {
|
|
13081
|
+
if (this.model) {
|
|
13082
|
+
throw new ModelConfigError(
|
|
13083
|
+
"Cannot change output after model is built. Use setOutput() before training.",
|
|
13084
|
+
{ model: this.config.name }
|
|
13085
|
+
);
|
|
13086
|
+
}
|
|
13087
|
+
if (outputConfig.activation) {
|
|
13088
|
+
this.config.modelConfig.outputActivation = outputConfig.activation;
|
|
13089
|
+
}
|
|
13090
|
+
if (outputConfig.units) {
|
|
13091
|
+
this.config.modelConfig.outputUnits = outputConfig.units;
|
|
13092
|
+
}
|
|
13093
|
+
if (outputConfig.loss) {
|
|
13094
|
+
this.config.modelConfig.loss = outputConfig.loss;
|
|
13095
|
+
}
|
|
13096
|
+
if (outputConfig.metrics) {
|
|
13097
|
+
this.config.modelConfig.metrics = outputConfig.metrics;
|
|
13098
|
+
}
|
|
13099
|
+
}
|
|
13100
|
+
/**
|
|
13101
|
+
* Get model architecture summary
|
|
13102
|
+
*/
|
|
13103
|
+
getArchitecture() {
|
|
13104
|
+
return {
|
|
13105
|
+
inputFeatures: this.config.features,
|
|
13106
|
+
hiddenLayers: this.config.modelConfig.layers.map((layer, index) => ({
|
|
13107
|
+
index,
|
|
13108
|
+
units: layer.units,
|
|
13109
|
+
activation: layer.activation || "relu",
|
|
13110
|
+
dropout: layer.dropout || 0,
|
|
13111
|
+
batchNormalization: layer.batchNormalization || false
|
|
13112
|
+
})),
|
|
13113
|
+
outputLayer: {
|
|
13114
|
+
units: this.config.modelConfig.outputUnits,
|
|
13115
|
+
activation: this.config.modelConfig.outputActivation
|
|
13116
|
+
},
|
|
13117
|
+
totalParameters: this._countParameters(),
|
|
13118
|
+
loss: this.config.modelConfig.loss,
|
|
13119
|
+
metrics: this.config.modelConfig.metrics
|
|
13120
|
+
};
|
|
13121
|
+
}
|
|
13122
|
+
/**
|
|
13123
|
+
* Train with early stopping callback
|
|
13124
|
+
* @param {Array} data - Training data
|
|
13125
|
+
* @param {Object} earlyStoppingConfig - Early stopping configuration
|
|
13126
|
+
* @returns {Object} Training results
|
|
13127
|
+
*/
|
|
13128
|
+
async trainWithEarlyStopping(data, earlyStoppingConfig = {}) {
|
|
13129
|
+
const {
|
|
13130
|
+
patience = 10,
|
|
13131
|
+
minDelta = 1e-3,
|
|
13132
|
+
monitor = "val_loss",
|
|
13133
|
+
restoreBestWeights = true
|
|
13134
|
+
} = earlyStoppingConfig;
|
|
13135
|
+
const { xs, ys } = this._prepareData(data);
|
|
13136
|
+
if (!this.model) {
|
|
13137
|
+
this.buildModel();
|
|
13138
|
+
}
|
|
13139
|
+
let bestValue = Infinity;
|
|
13140
|
+
let patienceCounter = 0;
|
|
13141
|
+
let bestWeights = null;
|
|
13142
|
+
const callbacks = {
|
|
13143
|
+
onEpochEnd: async (epoch, logs) => {
|
|
13144
|
+
const monitorValue = logs[monitor] || logs.loss;
|
|
13145
|
+
if (this.config.verbose && epoch % 10 === 0) {
|
|
13146
|
+
console.log(`[MLPlugin] ${this.config.name} - Epoch ${epoch}: ${monitor}=${monitorValue.toFixed(4)}`);
|
|
13147
|
+
}
|
|
13148
|
+
if (monitorValue < bestValue - minDelta) {
|
|
13149
|
+
bestValue = monitorValue;
|
|
13150
|
+
patienceCounter = 0;
|
|
13151
|
+
if (restoreBestWeights) {
|
|
13152
|
+
bestWeights = await this.model.getWeights();
|
|
13153
|
+
}
|
|
13154
|
+
} else {
|
|
13155
|
+
patienceCounter++;
|
|
13156
|
+
if (patienceCounter >= patience) {
|
|
13157
|
+
if (this.config.verbose) {
|
|
13158
|
+
console.log(`[MLPlugin] ${this.config.name} - Early stopping at epoch ${epoch}`);
|
|
13159
|
+
}
|
|
13160
|
+
this.model.stopTraining = true;
|
|
13161
|
+
}
|
|
13162
|
+
}
|
|
13163
|
+
}
|
|
13164
|
+
};
|
|
13165
|
+
const history = await this.model.fit(xs, ys, {
|
|
13166
|
+
epochs: this.config.modelConfig.epochs,
|
|
13167
|
+
batchSize: this.config.modelConfig.batchSize,
|
|
13168
|
+
validationSplit: this.config.modelConfig.validationSplit,
|
|
13169
|
+
verbose: this.config.verbose ? 1 : 0,
|
|
13170
|
+
callbacks
|
|
13171
|
+
});
|
|
13172
|
+
if (restoreBestWeights && bestWeights) {
|
|
13173
|
+
this.model.setWeights(bestWeights);
|
|
13174
|
+
}
|
|
13175
|
+
this.isTrained = true;
|
|
13176
|
+
this.stats.trainedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
13177
|
+
this.stats.samples = data.length;
|
|
13178
|
+
this.stats.loss = history.history.loss[history.history.loss.length - 1];
|
|
13179
|
+
xs.dispose();
|
|
13180
|
+
ys.dispose();
|
|
13181
|
+
return {
|
|
13182
|
+
loss: this.stats.loss,
|
|
13183
|
+
epochs: history.epoch.length,
|
|
13184
|
+
samples: this.stats.samples,
|
|
13185
|
+
stoppedEarly: history.epoch.length < this.config.modelConfig.epochs
|
|
13186
|
+
};
|
|
13187
|
+
}
|
|
13188
|
+
/**
|
|
13189
|
+
* Export model with neural network-specific data
|
|
13190
|
+
*/
|
|
13191
|
+
async export() {
|
|
13192
|
+
const baseExport = await super.export();
|
|
13193
|
+
return {
|
|
13194
|
+
...baseExport,
|
|
13195
|
+
type: "neural-network",
|
|
13196
|
+
architecture: this.getArchitecture()
|
|
13197
|
+
};
|
|
13198
|
+
}
|
|
13199
|
+
}
|
|
13200
|
+
|
|
13201
|
+
class MLPlugin extends Plugin {
|
|
13202
|
+
constructor(options = {}) {
|
|
13203
|
+
super(options);
|
|
13204
|
+
this.config = {
|
|
13205
|
+
models: options.models || {},
|
|
13206
|
+
verbose: options.verbose || false,
|
|
13207
|
+
minTrainingSamples: options.minTrainingSamples || 10
|
|
13208
|
+
};
|
|
13209
|
+
requirePluginDependency("@tensorflow/tfjs-node", "MLPlugin");
|
|
13210
|
+
this.models = {};
|
|
13211
|
+
this.training = /* @__PURE__ */ new Map();
|
|
13212
|
+
this.insertCounters = /* @__PURE__ */ new Map();
|
|
13213
|
+
this.intervals = [];
|
|
13214
|
+
this.stats = {
|
|
13215
|
+
totalTrainings: 0,
|
|
13216
|
+
totalPredictions: 0,
|
|
13217
|
+
totalErrors: 0,
|
|
13218
|
+
startedAt: null
|
|
13219
|
+
};
|
|
13220
|
+
}
|
|
13221
|
+
/**
|
|
13222
|
+
* Install the plugin
|
|
13223
|
+
*/
|
|
13224
|
+
async onInstall() {
|
|
13225
|
+
if (this.config.verbose) {
|
|
13226
|
+
console.log("[MLPlugin] Installing ML Plugin...");
|
|
13227
|
+
}
|
|
13228
|
+
for (const [modelName, modelConfig] of Object.entries(this.config.models)) {
|
|
13229
|
+
this._validateModelConfig(modelName, modelConfig);
|
|
13230
|
+
}
|
|
13231
|
+
for (const [modelName, modelConfig] of Object.entries(this.config.models)) {
|
|
13232
|
+
await this._initializeModel(modelName, modelConfig);
|
|
13233
|
+
}
|
|
13234
|
+
for (const [modelName, modelConfig] of Object.entries(this.config.models)) {
|
|
13235
|
+
if (modelConfig.autoTrain) {
|
|
13236
|
+
this._setupAutoTraining(modelName, modelConfig);
|
|
13237
|
+
}
|
|
13238
|
+
}
|
|
13239
|
+
this.stats.startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
13240
|
+
if (this.config.verbose) {
|
|
13241
|
+
console.log(`[MLPlugin] Installed with ${Object.keys(this.models).length} models`);
|
|
13242
|
+
}
|
|
13243
|
+
this.emit("installed", {
|
|
13244
|
+
plugin: "MLPlugin",
|
|
13245
|
+
models: Object.keys(this.models)
|
|
13246
|
+
});
|
|
13247
|
+
}
|
|
13248
|
+
/**
|
|
13249
|
+
* Start the plugin
|
|
13250
|
+
*/
|
|
13251
|
+
async onStart() {
|
|
13252
|
+
for (const modelName of Object.keys(this.models)) {
|
|
13253
|
+
await this._loadModel(modelName);
|
|
13254
|
+
}
|
|
13255
|
+
if (this.config.verbose) {
|
|
13256
|
+
console.log("[MLPlugin] Started");
|
|
13257
|
+
}
|
|
13258
|
+
}
|
|
13259
|
+
/**
|
|
13260
|
+
* Stop the plugin
|
|
13261
|
+
*/
|
|
13262
|
+
async onStop() {
|
|
13263
|
+
for (const handle of this.intervals) {
|
|
13264
|
+
clearInterval(handle);
|
|
13265
|
+
}
|
|
13266
|
+
this.intervals = [];
|
|
13267
|
+
for (const [modelName, model] of Object.entries(this.models)) {
|
|
13268
|
+
if (model && model.dispose) {
|
|
13269
|
+
model.dispose();
|
|
13270
|
+
}
|
|
13271
|
+
}
|
|
13272
|
+
if (this.config.verbose) {
|
|
13273
|
+
console.log("[MLPlugin] Stopped");
|
|
13274
|
+
}
|
|
13275
|
+
}
|
|
13276
|
+
/**
|
|
13277
|
+
* Uninstall the plugin
|
|
13278
|
+
*/
|
|
13279
|
+
async onUninstall(options = {}) {
|
|
13280
|
+
await this.onStop();
|
|
13281
|
+
if (options.purgeData) {
|
|
13282
|
+
for (const modelName of Object.keys(this.models)) {
|
|
13283
|
+
await this._deleteModel(modelName);
|
|
13284
|
+
}
|
|
13285
|
+
if (this.config.verbose) {
|
|
13286
|
+
console.log("[MLPlugin] Purged all model data");
|
|
13287
|
+
}
|
|
13288
|
+
}
|
|
13289
|
+
}
|
|
13290
|
+
/**
|
|
13291
|
+
* Validate model configuration
|
|
13292
|
+
* @private
|
|
13293
|
+
*/
|
|
13294
|
+
_validateModelConfig(modelName, config) {
|
|
13295
|
+
const validTypes = ["regression", "classification", "timeseries", "neural-network"];
|
|
13296
|
+
if (!config.type || !validTypes.includes(config.type)) {
|
|
13297
|
+
throw new ModelConfigError(
|
|
13298
|
+
`Model "${modelName}" must have a valid type: ${validTypes.join(", ")}`,
|
|
13299
|
+
{ modelName, type: config.type, validTypes }
|
|
13300
|
+
);
|
|
13301
|
+
}
|
|
13302
|
+
if (!config.resource) {
|
|
13303
|
+
throw new ModelConfigError(
|
|
13304
|
+
`Model "${modelName}" must specify a resource`,
|
|
13305
|
+
{ modelName }
|
|
13306
|
+
);
|
|
13307
|
+
}
|
|
13308
|
+
if (!config.features || !Array.isArray(config.features) || config.features.length === 0) {
|
|
13309
|
+
throw new ModelConfigError(
|
|
13310
|
+
`Model "${modelName}" must specify at least one feature`,
|
|
13311
|
+
{ modelName, features: config.features }
|
|
13312
|
+
);
|
|
13313
|
+
}
|
|
13314
|
+
if (!config.target) {
|
|
13315
|
+
throw new ModelConfigError(
|
|
13316
|
+
`Model "${modelName}" must specify a target field`,
|
|
13317
|
+
{ modelName }
|
|
13318
|
+
);
|
|
13319
|
+
}
|
|
13320
|
+
}
|
|
13321
|
+
/**
|
|
13322
|
+
* Initialize a model instance
|
|
13323
|
+
* @private
|
|
13324
|
+
*/
|
|
13325
|
+
async _initializeModel(modelName, config) {
|
|
13326
|
+
const modelOptions = {
|
|
13327
|
+
name: modelName,
|
|
13328
|
+
resource: config.resource,
|
|
13329
|
+
features: config.features,
|
|
13330
|
+
target: config.target,
|
|
13331
|
+
modelConfig: config.modelConfig || {},
|
|
13332
|
+
verbose: this.config.verbose
|
|
13333
|
+
};
|
|
13334
|
+
try {
|
|
13335
|
+
switch (config.type) {
|
|
13336
|
+
case "regression":
|
|
13337
|
+
this.models[modelName] = new RegressionModel(modelOptions);
|
|
13338
|
+
break;
|
|
13339
|
+
case "classification":
|
|
13340
|
+
this.models[modelName] = new ClassificationModel(modelOptions);
|
|
13341
|
+
break;
|
|
13342
|
+
case "timeseries":
|
|
13343
|
+
this.models[modelName] = new TimeSeriesModel(modelOptions);
|
|
13344
|
+
break;
|
|
13345
|
+
case "neural-network":
|
|
13346
|
+
this.models[modelName] = new NeuralNetworkModel(modelOptions);
|
|
13347
|
+
break;
|
|
13348
|
+
default:
|
|
13349
|
+
throw new ModelConfigError(
|
|
13350
|
+
`Unknown model type: ${config.type}`,
|
|
13351
|
+
{ modelName, type: config.type }
|
|
13352
|
+
);
|
|
13353
|
+
}
|
|
13354
|
+
if (this.config.verbose) {
|
|
13355
|
+
console.log(`[MLPlugin] Initialized model "${modelName}" (${config.type})`);
|
|
13356
|
+
}
|
|
13357
|
+
} catch (error) {
|
|
13358
|
+
console.error(`[MLPlugin] Failed to initialize model "${modelName}":`, error.message);
|
|
13359
|
+
throw error;
|
|
13360
|
+
}
|
|
13361
|
+
}
|
|
13362
|
+
/**
|
|
13363
|
+
* Setup auto-training for a model
|
|
13364
|
+
* @private
|
|
13365
|
+
*/
|
|
13366
|
+
_setupAutoTraining(modelName, config) {
|
|
13367
|
+
const resource = this.database.resources[config.resource];
|
|
13368
|
+
if (!resource) {
|
|
13369
|
+
console.warn(`[MLPlugin] Resource "${config.resource}" not found for model "${modelName}"`);
|
|
13370
|
+
return;
|
|
13371
|
+
}
|
|
13372
|
+
this.insertCounters.set(modelName, 0);
|
|
13373
|
+
if (config.trainAfterInserts && config.trainAfterInserts > 0) {
|
|
13374
|
+
this.addMiddleware(resource, "insert", async (next, data, options) => {
|
|
13375
|
+
const result = await next(data, options);
|
|
13376
|
+
const currentCount = this.insertCounters.get(modelName) || 0;
|
|
13377
|
+
this.insertCounters.set(modelName, currentCount + 1);
|
|
13378
|
+
if (this.insertCounters.get(modelName) >= config.trainAfterInserts) {
|
|
13379
|
+
if (this.config.verbose) {
|
|
13380
|
+
console.log(`[MLPlugin] Auto-training "${modelName}" after ${config.trainAfterInserts} inserts`);
|
|
13381
|
+
}
|
|
13382
|
+
this.insertCounters.set(modelName, 0);
|
|
13383
|
+
this.train(modelName).catch((err) => {
|
|
13384
|
+
console.error(`[MLPlugin] Auto-training failed for "${modelName}":`, err.message);
|
|
13385
|
+
});
|
|
13386
|
+
}
|
|
13387
|
+
return result;
|
|
13388
|
+
});
|
|
13389
|
+
}
|
|
13390
|
+
if (config.trainInterval && config.trainInterval > 0) {
|
|
13391
|
+
const handle = setInterval(async () => {
|
|
13392
|
+
if (this.config.verbose) {
|
|
13393
|
+
console.log(`[MLPlugin] Auto-training "${modelName}" (interval: ${config.trainInterval}ms)`);
|
|
13394
|
+
}
|
|
13395
|
+
try {
|
|
13396
|
+
await this.train(modelName);
|
|
13397
|
+
} catch (error) {
|
|
13398
|
+
console.error(`[MLPlugin] Auto-training failed for "${modelName}":`, error.message);
|
|
13399
|
+
}
|
|
13400
|
+
}, config.trainInterval);
|
|
13401
|
+
this.intervals.push(handle);
|
|
13402
|
+
if (this.config.verbose) {
|
|
13403
|
+
console.log(`[MLPlugin] Setup interval training for "${modelName}" (every ${config.trainInterval}ms)`);
|
|
13404
|
+
}
|
|
13405
|
+
}
|
|
13406
|
+
}
|
|
13407
|
+
/**
|
|
13408
|
+
* Train a model
|
|
13409
|
+
* @param {string} modelName - Model name
|
|
13410
|
+
* @param {Object} options - Training options
|
|
13411
|
+
* @returns {Object} Training results
|
|
13412
|
+
*/
|
|
13413
|
+
async train(modelName, options = {}) {
|
|
13414
|
+
const model = this.models[modelName];
|
|
13415
|
+
if (!model) {
|
|
13416
|
+
throw new ModelNotFoundError(
|
|
13417
|
+
`Model "${modelName}" not found`,
|
|
13418
|
+
{ modelName, availableModels: Object.keys(this.models) }
|
|
13419
|
+
);
|
|
13420
|
+
}
|
|
13421
|
+
if (this.training.get(modelName)) {
|
|
13422
|
+
if (this.config.verbose) {
|
|
13423
|
+
console.log(`[MLPlugin] Model "${modelName}" is already training, skipping...`);
|
|
13424
|
+
}
|
|
13425
|
+
return { skipped: true, reason: "already_training" };
|
|
13426
|
+
}
|
|
13427
|
+
this.training.set(modelName, true);
|
|
13428
|
+
try {
|
|
13429
|
+
const modelConfig = this.config.models[modelName];
|
|
13430
|
+
const resource = this.database.resources[modelConfig.resource];
|
|
13431
|
+
if (!resource) {
|
|
13432
|
+
throw new ModelNotFoundError(
|
|
13433
|
+
`Resource "${modelConfig.resource}" not found`,
|
|
13434
|
+
{ modelName, resource: modelConfig.resource }
|
|
13435
|
+
);
|
|
13436
|
+
}
|
|
13437
|
+
if (this.config.verbose) {
|
|
13438
|
+
console.log(`[MLPlugin] Fetching training data for "${modelName}"...`);
|
|
13439
|
+
}
|
|
13440
|
+
const [ok, err, data] = await tryFn(() => resource.list());
|
|
13441
|
+
if (!ok) {
|
|
13442
|
+
throw new TrainingError(
|
|
13443
|
+
`Failed to fetch training data: ${err.message}`,
|
|
13444
|
+
{ modelName, resource: modelConfig.resource, originalError: err.message }
|
|
13445
|
+
);
|
|
13446
|
+
}
|
|
13447
|
+
if (!data || data.length < this.config.minTrainingSamples) {
|
|
13448
|
+
throw new TrainingError(
|
|
13449
|
+
`Insufficient training data: ${data?.length || 0} samples (minimum: ${this.config.minTrainingSamples})`,
|
|
13450
|
+
{ modelName, samples: data?.length || 0, minimum: this.config.minTrainingSamples }
|
|
13451
|
+
);
|
|
13452
|
+
}
|
|
13453
|
+
if (this.config.verbose) {
|
|
13454
|
+
console.log(`[MLPlugin] Training "${modelName}" with ${data.length} samples...`);
|
|
13455
|
+
}
|
|
13456
|
+
const result = await model.train(data);
|
|
13457
|
+
await this._saveModel(modelName);
|
|
13458
|
+
this.stats.totalTrainings++;
|
|
13459
|
+
if (this.config.verbose) {
|
|
13460
|
+
console.log(`[MLPlugin] Training completed for "${modelName}":`, result);
|
|
13461
|
+
}
|
|
13462
|
+
this.emit("modelTrained", {
|
|
13463
|
+
modelName,
|
|
13464
|
+
type: modelConfig.type,
|
|
13465
|
+
result
|
|
13466
|
+
});
|
|
13467
|
+
return result;
|
|
13468
|
+
} catch (error) {
|
|
13469
|
+
this.stats.totalErrors++;
|
|
13470
|
+
if (error instanceof MLError) {
|
|
13471
|
+
throw error;
|
|
13472
|
+
}
|
|
13473
|
+
throw new TrainingError(
|
|
13474
|
+
`Training failed for "${modelName}": ${error.message}`,
|
|
13475
|
+
{ modelName, originalError: error.message }
|
|
13476
|
+
);
|
|
13477
|
+
} finally {
|
|
13478
|
+
this.training.set(modelName, false);
|
|
13479
|
+
}
|
|
13480
|
+
}
|
|
13481
|
+
/**
|
|
13482
|
+
* Make a prediction
|
|
13483
|
+
* @param {string} modelName - Model name
|
|
13484
|
+
* @param {Object|Array} input - Input data (object for single prediction, array for time series)
|
|
13485
|
+
* @returns {Object} Prediction result
|
|
13486
|
+
*/
|
|
13487
|
+
async predict(modelName, input) {
|
|
13488
|
+
const model = this.models[modelName];
|
|
13489
|
+
if (!model) {
|
|
13490
|
+
throw new ModelNotFoundError(
|
|
13491
|
+
`Model "${modelName}" not found`,
|
|
13492
|
+
{ modelName, availableModels: Object.keys(this.models) }
|
|
13493
|
+
);
|
|
13494
|
+
}
|
|
13495
|
+
try {
|
|
13496
|
+
const result = await model.predict(input);
|
|
13497
|
+
this.stats.totalPredictions++;
|
|
13498
|
+
this.emit("prediction", {
|
|
13499
|
+
modelName,
|
|
13500
|
+
input,
|
|
13501
|
+
result
|
|
13502
|
+
});
|
|
13503
|
+
return result;
|
|
13504
|
+
} catch (error) {
|
|
13505
|
+
this.stats.totalErrors++;
|
|
13506
|
+
throw error;
|
|
13507
|
+
}
|
|
13508
|
+
}
|
|
13509
|
+
/**
|
|
13510
|
+
* Make predictions for multiple inputs
|
|
13511
|
+
* @param {string} modelName - Model name
|
|
13512
|
+
* @param {Array} inputs - Array of input objects
|
|
13513
|
+
* @returns {Array} Array of prediction results
|
|
13514
|
+
*/
|
|
13515
|
+
async predictBatch(modelName, inputs) {
|
|
13516
|
+
const model = this.models[modelName];
|
|
13517
|
+
if (!model) {
|
|
13518
|
+
throw new ModelNotFoundError(
|
|
13519
|
+
`Model "${modelName}" not found`,
|
|
13520
|
+
{ modelName, availableModels: Object.keys(this.models) }
|
|
13521
|
+
);
|
|
13522
|
+
}
|
|
13523
|
+
return await model.predictBatch(inputs);
|
|
13524
|
+
}
|
|
13525
|
+
/**
|
|
13526
|
+
* Retrain a model (reset and train from scratch)
|
|
13527
|
+
* @param {string} modelName - Model name
|
|
13528
|
+
* @param {Object} options - Options
|
|
13529
|
+
* @returns {Object} Training results
|
|
13530
|
+
*/
|
|
13531
|
+
async retrain(modelName, options = {}) {
|
|
13532
|
+
const model = this.models[modelName];
|
|
13533
|
+
if (!model) {
|
|
13534
|
+
throw new ModelNotFoundError(
|
|
13535
|
+
`Model "${modelName}" not found`,
|
|
13536
|
+
{ modelName, availableModels: Object.keys(this.models) }
|
|
13537
|
+
);
|
|
13538
|
+
}
|
|
13539
|
+
if (model.dispose) {
|
|
13540
|
+
model.dispose();
|
|
13541
|
+
}
|
|
13542
|
+
const modelConfig = this.config.models[modelName];
|
|
13543
|
+
await this._initializeModel(modelName, modelConfig);
|
|
13544
|
+
return await this.train(modelName, options);
|
|
13545
|
+
}
|
|
13546
|
+
/**
|
|
13547
|
+
* Get model statistics
|
|
13548
|
+
* @param {string} modelName - Model name
|
|
13549
|
+
* @returns {Object} Model stats
|
|
13550
|
+
*/
|
|
13551
|
+
getModelStats(modelName) {
|
|
13552
|
+
const model = this.models[modelName];
|
|
13553
|
+
if (!model) {
|
|
13554
|
+
throw new ModelNotFoundError(
|
|
13555
|
+
`Model "${modelName}" not found`,
|
|
13556
|
+
{ modelName, availableModels: Object.keys(this.models) }
|
|
13557
|
+
);
|
|
13558
|
+
}
|
|
13559
|
+
return model.getStats();
|
|
13560
|
+
}
|
|
13561
|
+
/**
|
|
13562
|
+
* Get plugin statistics
|
|
13563
|
+
* @returns {Object} Plugin stats
|
|
13564
|
+
*/
|
|
13565
|
+
getStats() {
|
|
13566
|
+
return {
|
|
13567
|
+
...this.stats,
|
|
13568
|
+
models: Object.keys(this.models).length,
|
|
13569
|
+
trainedModels: Object.values(this.models).filter((m) => m.isTrained).length
|
|
13570
|
+
};
|
|
13571
|
+
}
|
|
13572
|
+
/**
|
|
13573
|
+
* Export a model
|
|
13574
|
+
* @param {string} modelName - Model name
|
|
13575
|
+
* @returns {Object} Serialized model
|
|
13576
|
+
*/
|
|
13577
|
+
async exportModel(modelName) {
|
|
13578
|
+
const model = this.models[modelName];
|
|
13579
|
+
if (!model) {
|
|
13580
|
+
throw new ModelNotFoundError(
|
|
13581
|
+
`Model "${modelName}" not found`,
|
|
13582
|
+
{ modelName, availableModels: Object.keys(this.models) }
|
|
13583
|
+
);
|
|
13584
|
+
}
|
|
13585
|
+
return await model.export();
|
|
13586
|
+
}
|
|
13587
|
+
/**
|
|
13588
|
+
* Import a model
|
|
13589
|
+
* @param {string} modelName - Model name
|
|
13590
|
+
* @param {Object} data - Serialized model data
|
|
13591
|
+
*/
|
|
13592
|
+
async importModel(modelName, data) {
|
|
13593
|
+
const model = this.models[modelName];
|
|
13594
|
+
if (!model) {
|
|
13595
|
+
throw new ModelNotFoundError(
|
|
13596
|
+
`Model "${modelName}" not found`,
|
|
13597
|
+
{ modelName, availableModels: Object.keys(this.models) }
|
|
13598
|
+
);
|
|
13599
|
+
}
|
|
13600
|
+
await model.import(data);
|
|
13601
|
+
await this._saveModel(modelName);
|
|
13602
|
+
if (this.config.verbose) {
|
|
13603
|
+
console.log(`[MLPlugin] Imported model "${modelName}"`);
|
|
13604
|
+
}
|
|
13605
|
+
}
|
|
13606
|
+
/**
|
|
13607
|
+
* Save model to plugin storage
|
|
13608
|
+
* @private
|
|
13609
|
+
*/
|
|
13610
|
+
async _saveModel(modelName) {
|
|
13611
|
+
try {
|
|
13612
|
+
const storage = this.getStorage();
|
|
13613
|
+
const exportedModel = await this.models[modelName].export();
|
|
13614
|
+
if (!exportedModel) {
|
|
13615
|
+
if (this.config.verbose) {
|
|
13616
|
+
console.log(`[MLPlugin] Model "${modelName}" not trained, skipping save`);
|
|
13617
|
+
}
|
|
13618
|
+
return;
|
|
13619
|
+
}
|
|
13620
|
+
await storage.patch(`model_${modelName}`, {
|
|
13621
|
+
modelName,
|
|
13622
|
+
data: JSON.stringify(exportedModel),
|
|
13623
|
+
savedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13624
|
+
});
|
|
13625
|
+
if (this.config.verbose) {
|
|
13626
|
+
console.log(`[MLPlugin] Saved model "${modelName}" to plugin storage`);
|
|
13627
|
+
}
|
|
13628
|
+
} catch (error) {
|
|
13629
|
+
console.error(`[MLPlugin] Failed to save model "${modelName}":`, error.message);
|
|
13630
|
+
}
|
|
13631
|
+
}
|
|
13632
|
+
/**
|
|
13633
|
+
* Load model from plugin storage
|
|
13634
|
+
* @private
|
|
13635
|
+
*/
|
|
13636
|
+
async _loadModel(modelName) {
|
|
13637
|
+
try {
|
|
13638
|
+
const storage = this.getStorage();
|
|
13639
|
+
const [ok, err, record] = await tryFn(() => storage.get(`model_${modelName}`));
|
|
13640
|
+
if (!ok || !record) {
|
|
13641
|
+
if (this.config.verbose) {
|
|
13642
|
+
console.log(`[MLPlugin] No saved model found for "${modelName}"`);
|
|
13643
|
+
}
|
|
13644
|
+
return;
|
|
13645
|
+
}
|
|
13646
|
+
const modelData = JSON.parse(record.data);
|
|
13647
|
+
await this.models[modelName].import(modelData);
|
|
13648
|
+
if (this.config.verbose) {
|
|
13649
|
+
console.log(`[MLPlugin] Loaded model "${modelName}" from plugin storage`);
|
|
13650
|
+
}
|
|
13651
|
+
} catch (error) {
|
|
13652
|
+
console.error(`[MLPlugin] Failed to load model "${modelName}":`, error.message);
|
|
13653
|
+
}
|
|
13654
|
+
}
|
|
13655
|
+
/**
|
|
13656
|
+
* Delete model from plugin storage
|
|
13657
|
+
* @private
|
|
13658
|
+
*/
|
|
13659
|
+
async _deleteModel(modelName) {
|
|
13660
|
+
try {
|
|
13661
|
+
const storage = this.getStorage();
|
|
13662
|
+
await storage.delete(`model_${modelName}`);
|
|
13663
|
+
if (this.config.verbose) {
|
|
13664
|
+
console.log(`[MLPlugin] Deleted model "${modelName}" from plugin storage`);
|
|
13665
|
+
}
|
|
13666
|
+
} catch (error) {
|
|
13667
|
+
if (this.config.verbose) {
|
|
13668
|
+
console.log(`[MLPlugin] Could not delete model "${modelName}": ${error.message}`);
|
|
13669
|
+
}
|
|
13670
|
+
}
|
|
13671
|
+
}
|
|
13672
|
+
}
|
|
13673
|
+
|
|
11906
13674
|
class SqsConsumer {
|
|
11907
13675
|
constructor({ queueUrl, onMessage, onError, poolingInterval = 5e3, maxMessages = 10, region = "us-east-1", credentials, endpoint, driver = "sqs" }) {
|
|
11908
13676
|
this.driver = driver;
|
|
@@ -18434,6 +20202,7 @@ ${errorDetails}`,
|
|
|
18434
20202
|
events = {},
|
|
18435
20203
|
asyncEvents = true,
|
|
18436
20204
|
asyncPartitions = true,
|
|
20205
|
+
strictPartitions = false,
|
|
18437
20206
|
createdBy = "user"
|
|
18438
20207
|
} = config;
|
|
18439
20208
|
this.name = name;
|
|
@@ -18465,6 +20234,7 @@ ${errorDetails}`,
|
|
|
18465
20234
|
allNestedObjectsOptional,
|
|
18466
20235
|
asyncEvents,
|
|
18467
20236
|
asyncPartitions,
|
|
20237
|
+
strictPartitions,
|
|
18468
20238
|
createdBy
|
|
18469
20239
|
};
|
|
18470
20240
|
this.hooks = {
|
|
@@ -19217,17 +20987,31 @@ ${errorDetails}`,
|
|
|
19217
20987
|
throw errPut;
|
|
19218
20988
|
}
|
|
19219
20989
|
const insertedObject = await this.get(finalId);
|
|
19220
|
-
if (this.config.
|
|
19221
|
-
|
|
19222
|
-
this.createPartitionReferences(insertedObject)
|
|
20990
|
+
if (this.config.partitions && Object.keys(this.config.partitions).length > 0) {
|
|
20991
|
+
if (this.config.strictPartitions) {
|
|
20992
|
+
await this.createPartitionReferences(insertedObject);
|
|
20993
|
+
} else if (this.config.asyncPartitions) {
|
|
20994
|
+
setImmediate(() => {
|
|
20995
|
+
this.createPartitionReferences(insertedObject).catch((err) => {
|
|
20996
|
+
this.emit("partitionIndexError", {
|
|
20997
|
+
operation: "insert",
|
|
20998
|
+
id: finalId,
|
|
20999
|
+
error: err,
|
|
21000
|
+
message: err.message
|
|
21001
|
+
});
|
|
21002
|
+
});
|
|
21003
|
+
});
|
|
21004
|
+
} else {
|
|
21005
|
+
const [ok, err] = await tryFn(() => this.createPartitionReferences(insertedObject));
|
|
21006
|
+
if (!ok) {
|
|
19223
21007
|
this.emit("partitionIndexError", {
|
|
19224
21008
|
operation: "insert",
|
|
19225
21009
|
id: finalId,
|
|
19226
21010
|
error: err,
|
|
19227
21011
|
message: err.message
|
|
19228
21012
|
});
|
|
19229
|
-
}
|
|
19230
|
-
}
|
|
21013
|
+
}
|
|
21014
|
+
}
|
|
19231
21015
|
const nonPartitionHooks = this.hooks.afterInsert.filter(
|
|
19232
21016
|
(hook) => !hook.toString().includes("createPartitionReferences")
|
|
19233
21017
|
);
|
|
@@ -19522,17 +21306,31 @@ ${errorDetails}`,
|
|
|
19522
21306
|
body: finalBody,
|
|
19523
21307
|
behavior: this.behavior
|
|
19524
21308
|
});
|
|
19525
|
-
if (this.config.
|
|
19526
|
-
|
|
19527
|
-
this.handlePartitionReferenceUpdates(originalData, updatedData)
|
|
21309
|
+
if (this.config.partitions && Object.keys(this.config.partitions).length > 0) {
|
|
21310
|
+
if (this.config.strictPartitions) {
|
|
21311
|
+
await this.handlePartitionReferenceUpdates(originalData, updatedData);
|
|
21312
|
+
} else if (this.config.asyncPartitions) {
|
|
21313
|
+
setImmediate(() => {
|
|
21314
|
+
this.handlePartitionReferenceUpdates(originalData, updatedData).catch((err2) => {
|
|
21315
|
+
this.emit("partitionIndexError", {
|
|
21316
|
+
operation: "update",
|
|
21317
|
+
id,
|
|
21318
|
+
error: err2,
|
|
21319
|
+
message: err2.message
|
|
21320
|
+
});
|
|
21321
|
+
});
|
|
21322
|
+
});
|
|
21323
|
+
} else {
|
|
21324
|
+
const [ok2, err2] = await tryFn(() => this.handlePartitionReferenceUpdates(originalData, updatedData));
|
|
21325
|
+
if (!ok2) {
|
|
19528
21326
|
this.emit("partitionIndexError", {
|
|
19529
21327
|
operation: "update",
|
|
19530
21328
|
id,
|
|
19531
21329
|
error: err2,
|
|
19532
21330
|
message: err2.message
|
|
19533
21331
|
});
|
|
19534
|
-
}
|
|
19535
|
-
}
|
|
21332
|
+
}
|
|
21333
|
+
}
|
|
19536
21334
|
const nonPartitionHooks = this.hooks.afterUpdate.filter(
|
|
19537
21335
|
(hook) => !hook.toString().includes("handlePartitionReferenceUpdates")
|
|
19538
21336
|
);
|
|
@@ -19645,7 +21443,9 @@ ${errorDetails}`,
|
|
|
19645
21443
|
if (this.config.partitions && Object.keys(this.config.partitions).length > 0) {
|
|
19646
21444
|
const oldData = { ...currentData, id };
|
|
19647
21445
|
const newData = { ...mergedData, id };
|
|
19648
|
-
if (this.config.
|
|
21446
|
+
if (this.config.strictPartitions) {
|
|
21447
|
+
await this.handlePartitionReferenceUpdates(oldData, newData);
|
|
21448
|
+
} else if (this.config.asyncPartitions) {
|
|
19649
21449
|
setImmediate(() => {
|
|
19650
21450
|
this.handlePartitionReferenceUpdates(oldData, newData).catch((err) => {
|
|
19651
21451
|
this.emit("partitionIndexError", {
|
|
@@ -19775,7 +21575,9 @@ ${errorDetails}`,
|
|
|
19775
21575
|
}
|
|
19776
21576
|
const replacedObject = { id, ...validatedAttributes };
|
|
19777
21577
|
if (this.config.partitions && Object.keys(this.config.partitions).length > 0) {
|
|
19778
|
-
if (this.config.
|
|
21578
|
+
if (this.config.strictPartitions) {
|
|
21579
|
+
await this.handlePartitionReferenceUpdates({}, replacedObject);
|
|
21580
|
+
} else if (this.config.asyncPartitions) {
|
|
19779
21581
|
setImmediate(() => {
|
|
19780
21582
|
this.handlePartitionReferenceUpdates({}, replacedObject).catch((err) => {
|
|
19781
21583
|
this.emit("partitionIndexError", {
|
|
@@ -19915,17 +21717,31 @@ ${errorDetails}`,
|
|
|
19915
21717
|
});
|
|
19916
21718
|
const oldData = { ...originalData, id };
|
|
19917
21719
|
const newData = { ...validatedAttributes, id };
|
|
19918
|
-
if (this.config.
|
|
19919
|
-
|
|
19920
|
-
this.handlePartitionReferenceUpdates(oldData, newData)
|
|
21720
|
+
if (this.config.partitions && Object.keys(this.config.partitions).length > 0) {
|
|
21721
|
+
if (this.config.strictPartitions) {
|
|
21722
|
+
await this.handlePartitionReferenceUpdates(oldData, newData);
|
|
21723
|
+
} else if (this.config.asyncPartitions) {
|
|
21724
|
+
setImmediate(() => {
|
|
21725
|
+
this.handlePartitionReferenceUpdates(oldData, newData).catch((err2) => {
|
|
21726
|
+
this.emit("partitionIndexError", {
|
|
21727
|
+
operation: "updateConditional",
|
|
21728
|
+
id,
|
|
21729
|
+
error: err2,
|
|
21730
|
+
message: err2.message
|
|
21731
|
+
});
|
|
21732
|
+
});
|
|
21733
|
+
});
|
|
21734
|
+
} else {
|
|
21735
|
+
const [ok2, err2] = await tryFn(() => this.handlePartitionReferenceUpdates(oldData, newData));
|
|
21736
|
+
if (!ok2) {
|
|
19921
21737
|
this.emit("partitionIndexError", {
|
|
19922
21738
|
operation: "updateConditional",
|
|
19923
21739
|
id,
|
|
19924
21740
|
error: err2,
|
|
19925
21741
|
message: err2.message
|
|
19926
21742
|
});
|
|
19927
|
-
}
|
|
19928
|
-
}
|
|
21743
|
+
}
|
|
21744
|
+
}
|
|
19929
21745
|
const nonPartitionHooks = this.hooks.afterUpdate.filter(
|
|
19930
21746
|
(hook) => !hook.toString().includes("handlePartitionReferenceUpdates")
|
|
19931
21747
|
);
|
|
@@ -20001,17 +21817,31 @@ ${errorDetails}`,
|
|
|
20001
21817
|
operation: "delete",
|
|
20002
21818
|
id
|
|
20003
21819
|
});
|
|
20004
|
-
if (this.config.
|
|
20005
|
-
|
|
20006
|
-
this.deletePartitionReferences(objectData)
|
|
21820
|
+
if (this.config.partitions && Object.keys(this.config.partitions).length > 0 && objectData) {
|
|
21821
|
+
if (this.config.strictPartitions) {
|
|
21822
|
+
await this.deletePartitionReferences(objectData);
|
|
21823
|
+
} else if (this.config.asyncPartitions) {
|
|
21824
|
+
setImmediate(() => {
|
|
21825
|
+
this.deletePartitionReferences(objectData).catch((err3) => {
|
|
21826
|
+
this.emit("partitionIndexError", {
|
|
21827
|
+
operation: "delete",
|
|
21828
|
+
id,
|
|
21829
|
+
error: err3,
|
|
21830
|
+
message: err3.message
|
|
21831
|
+
});
|
|
21832
|
+
});
|
|
21833
|
+
});
|
|
21834
|
+
} else {
|
|
21835
|
+
const [ok3, err3] = await tryFn(() => this.deletePartitionReferences(objectData));
|
|
21836
|
+
if (!ok3) {
|
|
20007
21837
|
this.emit("partitionIndexError", {
|
|
20008
21838
|
operation: "delete",
|
|
20009
21839
|
id,
|
|
20010
21840
|
error: err3,
|
|
20011
21841
|
message: err3.message
|
|
20012
21842
|
});
|
|
20013
|
-
}
|
|
20014
|
-
}
|
|
21843
|
+
}
|
|
21844
|
+
}
|
|
20015
21845
|
const nonPartitionHooks = this.hooks.afterDelete.filter(
|
|
20016
21846
|
(hook) => !hook.toString().includes("deletePartitionReferences")
|
|
20017
21847
|
);
|
|
@@ -21382,10 +23212,13 @@ function validateResourceConfig(config) {
|
|
|
21382
23212
|
class Database extends EventEmitter {
|
|
21383
23213
|
constructor(options) {
|
|
21384
23214
|
super();
|
|
21385
|
-
this.id =
|
|
23215
|
+
this.id = (() => {
|
|
23216
|
+
const [ok, err, id] = tryFn(() => idGenerator(7));
|
|
23217
|
+
return ok && id ? id : `db-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
23218
|
+
})();
|
|
21386
23219
|
this.version = "1";
|
|
21387
23220
|
this.s3dbVersion = (() => {
|
|
21388
|
-
const [ok, err, version] = tryFn(() => true ? "
|
|
23221
|
+
const [ok, err, version] = tryFn(() => true ? "13.0.0" : "latest");
|
|
21389
23222
|
return ok ? version : "latest";
|
|
21390
23223
|
})();
|
|
21391
23224
|
this._resourcesMap = {};
|
|
@@ -21419,6 +23252,7 @@ class Database extends EventEmitter {
|
|
|
21419
23252
|
this.versioningEnabled = options.versioningEnabled || false;
|
|
21420
23253
|
this.persistHooks = options.persistHooks || false;
|
|
21421
23254
|
this.strictValidation = options.strictValidation !== false;
|
|
23255
|
+
this.strictHooks = options.strictHooks || false;
|
|
21422
23256
|
this._initHooks();
|
|
21423
23257
|
let connectionString = options.connectionString;
|
|
21424
23258
|
if (!connectionString && (options.bucket || options.accessKeyId || options.secretAccessKey)) {
|
|
@@ -21449,18 +23283,25 @@ class Database extends EventEmitter {
|
|
|
21449
23283
|
this.connectionString = connectionString;
|
|
21450
23284
|
this.bucket = this.client.bucket;
|
|
21451
23285
|
this.keyPrefix = this.client.keyPrefix;
|
|
21452
|
-
|
|
23286
|
+
this._registerExitListener();
|
|
23287
|
+
}
|
|
23288
|
+
/**
|
|
23289
|
+
* Register process exit listener for automatic cleanup
|
|
23290
|
+
* @private
|
|
23291
|
+
*/
|
|
23292
|
+
_registerExitListener() {
|
|
23293
|
+
if (!this._exitListenerRegistered && typeof process !== "undefined") {
|
|
21453
23294
|
this._exitListenerRegistered = true;
|
|
21454
|
-
|
|
21455
|
-
|
|
21456
|
-
|
|
21457
|
-
|
|
21458
|
-
|
|
21459
|
-
|
|
21460
|
-
}
|
|
23295
|
+
this._exitListener = async () => {
|
|
23296
|
+
if (this.isConnected()) {
|
|
23297
|
+
await tryFn(() => this.disconnect());
|
|
23298
|
+
}
|
|
23299
|
+
};
|
|
23300
|
+
process.on("exit", this._exitListener);
|
|
21461
23301
|
}
|
|
21462
23302
|
}
|
|
21463
23303
|
async connect() {
|
|
23304
|
+
this._registerExitListener();
|
|
21464
23305
|
await this.startPlugins();
|
|
21465
23306
|
let metadata = null;
|
|
21466
23307
|
let needsHealing = false;
|
|
@@ -22423,11 +24264,16 @@ class Database extends EventEmitter {
|
|
|
22423
24264
|
if (this.client && typeof this.client.removeAllListeners === "function") {
|
|
22424
24265
|
this.client.removeAllListeners();
|
|
22425
24266
|
}
|
|
24267
|
+
await this.emit("disconnected", /* @__PURE__ */ new Date());
|
|
22426
24268
|
this.removeAllListeners();
|
|
24269
|
+
if (this._exitListener && typeof process !== "undefined") {
|
|
24270
|
+
process.off("exit", this._exitListener);
|
|
24271
|
+
this._exitListener = null;
|
|
24272
|
+
this._exitListenerRegistered = false;
|
|
24273
|
+
}
|
|
22427
24274
|
this.savedMetadata = null;
|
|
22428
24275
|
this.plugins = {};
|
|
22429
24276
|
this.pluginList = [];
|
|
22430
|
-
this.emit("disconnected", /* @__PURE__ */ new Date());
|
|
22431
24277
|
});
|
|
22432
24278
|
}
|
|
22433
24279
|
/**
|
|
@@ -22531,6 +24377,13 @@ class Database extends EventEmitter {
|
|
|
22531
24377
|
const [ok, error] = await tryFn(() => hook({ database: this, ...context }));
|
|
22532
24378
|
if (!ok) {
|
|
22533
24379
|
this.emit("hookError", { event, error, context });
|
|
24380
|
+
if (this.strictHooks) {
|
|
24381
|
+
throw new DatabaseError(`Hook execution failed for event '${event}': ${error.message}`, {
|
|
24382
|
+
event,
|
|
24383
|
+
originalError: error,
|
|
24384
|
+
context
|
|
24385
|
+
});
|
|
24386
|
+
}
|
|
22534
24387
|
}
|
|
22535
24388
|
}
|
|
22536
24389
|
}
|
|
@@ -38858,30 +40711,42 @@ class MemoryClient extends EventEmitter {
|
|
|
38858
40711
|
const resourceStats = {};
|
|
38859
40712
|
for (const [resourceName, keys] of resourceMap.entries()) {
|
|
38860
40713
|
const records = [];
|
|
40714
|
+
const resource = database && database.resources && database.resources[resourceName];
|
|
38861
40715
|
for (const key of keys) {
|
|
38862
|
-
const obj = await this.getObject(key);
|
|
38863
40716
|
const idMatch = key.match(/\/id=([^/]+)/);
|
|
38864
40717
|
const recordId = idMatch ? idMatch[1] : null;
|
|
38865
|
-
|
|
38866
|
-
if (
|
|
38867
|
-
|
|
38868
|
-
|
|
38869
|
-
|
|
38870
|
-
|
|
38871
|
-
|
|
38872
|
-
chunks.push(chunk2);
|
|
40718
|
+
let record;
|
|
40719
|
+
if (resource && recordId) {
|
|
40720
|
+
try {
|
|
40721
|
+
record = await resource.get(recordId);
|
|
40722
|
+
} catch (err) {
|
|
40723
|
+
console.warn(`Failed to get record ${recordId} from resource ${resourceName}, using fallback`);
|
|
40724
|
+
record = null;
|
|
38873
40725
|
}
|
|
38874
|
-
|
|
38875
|
-
|
|
38876
|
-
|
|
38877
|
-
|
|
38878
|
-
|
|
38879
|
-
|
|
38880
|
-
|
|
40726
|
+
}
|
|
40727
|
+
if (!record) {
|
|
40728
|
+
const obj = await this.getObject(key);
|
|
40729
|
+
record = { ...obj.Metadata };
|
|
40730
|
+
if (recordId && !record.id) {
|
|
40731
|
+
record.id = recordId;
|
|
40732
|
+
}
|
|
40733
|
+
if (obj.Body) {
|
|
40734
|
+
const chunks = [];
|
|
40735
|
+
for await (const chunk2 of obj.Body) {
|
|
40736
|
+
chunks.push(chunk2);
|
|
40737
|
+
}
|
|
40738
|
+
const bodyBuffer = Buffer.concat(chunks);
|
|
40739
|
+
const bodyStr = bodyBuffer.toString("utf-8");
|
|
40740
|
+
if (bodyStr.startsWith("{") || bodyStr.startsWith("[")) {
|
|
40741
|
+
try {
|
|
40742
|
+
const bodyData = JSON.parse(bodyStr);
|
|
40743
|
+
Object.assign(record, bodyData);
|
|
40744
|
+
} catch {
|
|
40745
|
+
record._body = bodyStr;
|
|
40746
|
+
}
|
|
40747
|
+
} else if (bodyStr) {
|
|
38881
40748
|
record._body = bodyStr;
|
|
38882
40749
|
}
|
|
38883
|
-
} else if (bodyStr) {
|
|
38884
|
-
record._body = bodyStr;
|
|
38885
40750
|
}
|
|
38886
40751
|
}
|
|
38887
40752
|
records.push(record);
|
|
@@ -39942,5 +41807,5 @@ var metrics = /*#__PURE__*/Object.freeze({
|
|
|
39942
41807
|
silhouetteScore: silhouetteScore
|
|
39943
41808
|
});
|
|
39944
41809
|
|
|
39945
|
-
export { AVAILABLE_BEHAVIORS, AnalyticsNotEnabledError, ApiPlugin, AuditPlugin, AuthenticationError, BACKUP_DRIVERS, BackupPlugin, BaseBackupDriver, BaseError, BaseReplicator, BehaviorError, BigqueryReplicator, CONSUMER_DRIVERS, Cache, CachePlugin, S3Client as Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, DynamoDBReplicator, EncryptionError, ErrorMap, EventualConsistencyPlugin, Factory, FilesystemBackupDriver, FilesystemCache, FullTextPlugin, GeoPlugin, InvalidResourceItem, MemoryCache, MemoryClient, MemoryStorage, MetadataLimitError, MetricsPlugin, MissingMetadata, MongoDBReplicator, MultiBackupDriver, MySQLReplicator, NoSuchBucket, NoSuchKey, NotFound, PartitionAwareFilesystemCache, PartitionDriverError, PartitionError, PermissionError, PlanetScaleReplicator, Plugin, PluginError, PluginObject, PluginStorageError, PostgresReplicator, QueueConsumerPlugin, REPLICATOR_DRIVERS, RabbitMqConsumer, RelationPlugin, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, S3BackupDriver, S3Cache, S3Client, S3QueuePlugin, Database as S3db, S3dbError, S3dbReplicator, SchedulerPlugin, Schema, SchemaError, Seeder, SqsConsumer, SqsReplicator, StateMachinePlugin, StreamError, TTLPlugin, TfStatePlugin, TursoReplicator, UnknownError, ValidationError, Validator, VectorPlugin, WebhookReplicator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Memory, createBackupDriver, createConsumer, createReplicator, decode, decodeDecimal, decodeFixedPoint, decodeFixedPointBatch, decrypt, S3db as default, encode, encodeDecimal, encodeFixedPoint, encodeFixedPointBatch, encrypt, generateTypes, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, printTypes, sha256, streamToString, transformValue, tryFn, tryFnSync, validateBackupConfig, validateReplicatorConfig };
|
|
41810
|
+
export { AVAILABLE_BEHAVIORS, AnalyticsNotEnabledError, ApiPlugin, AuditPlugin, AuthenticationError, BACKUP_DRIVERS, BackupPlugin, BaseBackupDriver, BaseError, BaseReplicator, BehaviorError, BigqueryReplicator, CONSUMER_DRIVERS, Cache, CachePlugin, S3Client as Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, DynamoDBReplicator, EncryptionError, ErrorMap, EventualConsistencyPlugin, Factory, FilesystemBackupDriver, FilesystemCache, FullTextPlugin, GeoPlugin, InvalidResourceItem, MLPlugin, MemoryCache, MemoryClient, MemoryStorage, MetadataLimitError, MetricsPlugin, MissingMetadata, MongoDBReplicator, MultiBackupDriver, MySQLReplicator, NoSuchBucket, NoSuchKey, NotFound, PartitionAwareFilesystemCache, PartitionDriverError, PartitionError, PermissionError, PlanetScaleReplicator, Plugin, PluginError, PluginObject, PluginStorageError, PostgresReplicator, QueueConsumerPlugin, REPLICATOR_DRIVERS, RabbitMqConsumer, RelationPlugin, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, S3BackupDriver, S3Cache, S3Client, S3QueuePlugin, Database as S3db, S3dbError, S3dbReplicator, SchedulerPlugin, Schema, SchemaError, Seeder, SqsConsumer, SqsReplicator, StateMachinePlugin, StreamError, TTLPlugin, TfStatePlugin, TursoReplicator, UnknownError, ValidationError, Validator, VectorPlugin, WebhookReplicator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Memory, createBackupDriver, createConsumer, createReplicator, decode, decodeDecimal, decodeFixedPoint, decodeFixedPointBatch, decrypt, S3db as default, encode, encodeDecimal, encodeFixedPoint, encodeFixedPointBatch, encrypt, generateTypes, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, printTypes, sha256, streamToString, transformValue, tryFn, tryFnSync, validateBackupConfig, validateReplicatorConfig };
|
|
39946
41811
|
//# sourceMappingURL=s3db.es.js.map
|