s3db.js 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +620 -1222
- package/dist/s3db.cjs.js +3638 -502
- package/dist/s3db.cjs.min.js +1 -31
- package/dist/s3db.d.ts +891 -62
- package/dist/s3db.es.js +3636 -504
- package/dist/s3db.es.min.js +1 -31
- package/dist/s3db.iife.js +3638 -502
- package/dist/s3db.iife.min.js +1 -31
- package/package.json +33 -14
package/dist/s3db.cjs.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/* istanbul ignore file */
|
|
2
1
|
'use strict';
|
|
3
2
|
|
|
4
3
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
@@ -943,15 +942,24 @@ class Client extends EventEmitter {
|
|
|
943
942
|
if (errorClass) return new errorClass(data);
|
|
944
943
|
return error;
|
|
945
944
|
}
|
|
946
|
-
async putObject({ key, metadata, contentType, body, contentEncoding }) {
|
|
945
|
+
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength }) {
|
|
946
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
947
|
+
const stringMetadata = {};
|
|
948
|
+
if (metadata) {
|
|
949
|
+
for (const [k, v] of Object.entries(metadata)) {
|
|
950
|
+
const validKey = String(k).replace(/[^a-zA-Z0-9\-_]/g, "_");
|
|
951
|
+
stringMetadata[validKey] = String(v);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
947
954
|
const options2 = {
|
|
948
955
|
Bucket: this.config.bucket,
|
|
949
|
-
Key:
|
|
950
|
-
Metadata:
|
|
956
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key,
|
|
957
|
+
Metadata: stringMetadata,
|
|
951
958
|
Body: body || Buffer.alloc(0)
|
|
952
959
|
};
|
|
953
960
|
if (contentType !== void 0) options2.ContentType = contentType;
|
|
954
961
|
if (contentEncoding !== void 0) options2.ContentEncoding = contentEncoding;
|
|
962
|
+
if (contentLength !== void 0) options2.ContentLength = contentLength;
|
|
955
963
|
try {
|
|
956
964
|
const response = await this.sendCommand(new clientS3.PutObjectCommand(options2));
|
|
957
965
|
this.emit("putObject", response, options2);
|
|
@@ -964,9 +972,10 @@ class Client extends EventEmitter {
|
|
|
964
972
|
}
|
|
965
973
|
}
|
|
966
974
|
async getObject(key) {
|
|
975
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
967
976
|
const options2 = {
|
|
968
977
|
Bucket: this.config.bucket,
|
|
969
|
-
Key: path.join(
|
|
978
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
970
979
|
};
|
|
971
980
|
try {
|
|
972
981
|
const response = await this.sendCommand(new clientS3.GetObjectCommand(options2));
|
|
@@ -980,9 +989,10 @@ class Client extends EventEmitter {
|
|
|
980
989
|
}
|
|
981
990
|
}
|
|
982
991
|
async headObject(key) {
|
|
992
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
983
993
|
const options2 = {
|
|
984
994
|
Bucket: this.config.bucket,
|
|
985
|
-
Key:
|
|
995
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
986
996
|
};
|
|
987
997
|
try {
|
|
988
998
|
const response = await this.sendCommand(new clientS3.HeadObjectCommand(options2));
|
|
@@ -1024,9 +1034,10 @@ class Client extends EventEmitter {
|
|
|
1024
1034
|
}
|
|
1025
1035
|
}
|
|
1026
1036
|
async deleteObject(key) {
|
|
1037
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
1027
1038
|
const options2 = {
|
|
1028
1039
|
Bucket: this.config.bucket,
|
|
1029
|
-
Key:
|
|
1040
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
1030
1041
|
};
|
|
1031
1042
|
try {
|
|
1032
1043
|
const response = await this.sendCommand(new clientS3.DeleteObjectCommand(options2));
|
|
@@ -1040,13 +1051,14 @@ class Client extends EventEmitter {
|
|
|
1040
1051
|
}
|
|
1041
1052
|
}
|
|
1042
1053
|
async deleteObjects(keys) {
|
|
1054
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
1043
1055
|
const packages = lodashEs.chunk(keys, 1e3);
|
|
1044
1056
|
const { results, errors } = await promisePool.PromisePool.for(packages).withConcurrency(this.parallelism).process(async (keys2) => {
|
|
1045
1057
|
const options2 = {
|
|
1046
1058
|
Bucket: this.config.bucket,
|
|
1047
1059
|
Delete: {
|
|
1048
1060
|
Objects: keys2.map((key) => ({
|
|
1049
|
-
Key:
|
|
1061
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
1050
1062
|
}))
|
|
1051
1063
|
}
|
|
1052
1064
|
};
|
|
@@ -1074,12 +1086,13 @@ class Client extends EventEmitter {
|
|
|
1074
1086
|
* @returns {Promise<number>} Number of objects deleted
|
|
1075
1087
|
*/
|
|
1076
1088
|
async deleteAll({ prefix } = {}) {
|
|
1089
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
1077
1090
|
let continuationToken;
|
|
1078
1091
|
let totalDeleted = 0;
|
|
1079
1092
|
do {
|
|
1080
1093
|
const listCommand = new clientS3.ListObjectsV2Command({
|
|
1081
1094
|
Bucket: this.config.bucket,
|
|
1082
|
-
Prefix:
|
|
1095
|
+
Prefix: keyPrefix ? path.join(keyPrefix, prefix) : prefix,
|
|
1083
1096
|
ContinuationToken: continuationToken
|
|
1084
1097
|
});
|
|
1085
1098
|
const listResponse = await this.client.send(listCommand);
|
|
@@ -1221,6 +1234,10 @@ class Client extends EventEmitter {
|
|
|
1221
1234
|
prefix,
|
|
1222
1235
|
offset
|
|
1223
1236
|
});
|
|
1237
|
+
if (!continuationToken) {
|
|
1238
|
+
this.emit("getKeysPage", [], params);
|
|
1239
|
+
return [];
|
|
1240
|
+
}
|
|
1224
1241
|
}
|
|
1225
1242
|
while (truncated) {
|
|
1226
1243
|
const options2 = {
|
|
@@ -1233,8 +1250,8 @@ class Client extends EventEmitter {
|
|
|
1233
1250
|
}
|
|
1234
1251
|
truncated = res.IsTruncated || false;
|
|
1235
1252
|
continuationToken = res.NextContinuationToken;
|
|
1236
|
-
if (keys.length
|
|
1237
|
-
keys = keys.
|
|
1253
|
+
if (keys.length >= amount) {
|
|
1254
|
+
keys = keys.slice(0, amount);
|
|
1238
1255
|
break;
|
|
1239
1256
|
}
|
|
1240
1257
|
}
|
|
@@ -6006,6 +6023,16 @@ if (typeof Object.create === 'function'){
|
|
|
6006
6023
|
};
|
|
6007
6024
|
}
|
|
6008
6025
|
|
|
6026
|
+
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
6027
|
+
function getOwnPropertyDescriptors(obj) {
|
|
6028
|
+
var keys = Object.keys(obj);
|
|
6029
|
+
var descriptors = {};
|
|
6030
|
+
for (var i = 0; i < keys.length; i++) {
|
|
6031
|
+
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
6032
|
+
}
|
|
6033
|
+
return descriptors;
|
|
6034
|
+
};
|
|
6035
|
+
|
|
6009
6036
|
var formatRegExp = /%[sdj%]/g;
|
|
6010
6037
|
function format(f) {
|
|
6011
6038
|
if (!isString(f)) {
|
|
@@ -6491,6 +6518,64 @@ function hasOwnProperty(obj, prop) {
|
|
|
6491
6518
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
6492
6519
|
}
|
|
6493
6520
|
|
|
6521
|
+
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
6522
|
+
|
|
6523
|
+
function promisify(original) {
|
|
6524
|
+
if (typeof original !== 'function')
|
|
6525
|
+
throw new TypeError('The "original" argument must be of type Function');
|
|
6526
|
+
|
|
6527
|
+
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
6528
|
+
var fn = original[kCustomPromisifiedSymbol];
|
|
6529
|
+
if (typeof fn !== 'function') {
|
|
6530
|
+
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
6531
|
+
}
|
|
6532
|
+
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
6533
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
6534
|
+
});
|
|
6535
|
+
return fn;
|
|
6536
|
+
}
|
|
6537
|
+
|
|
6538
|
+
function fn() {
|
|
6539
|
+
var promiseResolve, promiseReject;
|
|
6540
|
+
var promise = new Promise(function (resolve, reject) {
|
|
6541
|
+
promiseResolve = resolve;
|
|
6542
|
+
promiseReject = reject;
|
|
6543
|
+
});
|
|
6544
|
+
|
|
6545
|
+
var args = [];
|
|
6546
|
+
for (var i = 0; i < arguments.length; i++) {
|
|
6547
|
+
args.push(arguments[i]);
|
|
6548
|
+
}
|
|
6549
|
+
args.push(function (err, value) {
|
|
6550
|
+
if (err) {
|
|
6551
|
+
promiseReject(err);
|
|
6552
|
+
} else {
|
|
6553
|
+
promiseResolve(value);
|
|
6554
|
+
}
|
|
6555
|
+
});
|
|
6556
|
+
|
|
6557
|
+
try {
|
|
6558
|
+
original.apply(this, args);
|
|
6559
|
+
} catch (err) {
|
|
6560
|
+
promiseReject(err);
|
|
6561
|
+
}
|
|
6562
|
+
|
|
6563
|
+
return promise;
|
|
6564
|
+
}
|
|
6565
|
+
|
|
6566
|
+
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
6567
|
+
|
|
6568
|
+
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
6569
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
6570
|
+
});
|
|
6571
|
+
return Object.defineProperties(
|
|
6572
|
+
fn,
|
|
6573
|
+
getOwnPropertyDescriptors(original)
|
|
6574
|
+
);
|
|
6575
|
+
}
|
|
6576
|
+
|
|
6577
|
+
promisify.custom = kCustomPromisifiedSymbol;
|
|
6578
|
+
|
|
6494
6579
|
function BufferList() {
|
|
6495
6580
|
this.head = null;
|
|
6496
6581
|
this.tail = null;
|
|
@@ -8720,41 +8805,41 @@ function getSizeBreakdown(mappedObject) {
|
|
|
8720
8805
|
}
|
|
8721
8806
|
|
|
8722
8807
|
const S3_METADATA_LIMIT_BYTES = 2048;
|
|
8723
|
-
async function handleInsert$
|
|
8808
|
+
async function handleInsert$4({ resource, data, mappedData }) {
|
|
8724
8809
|
const totalSize = calculateTotalSize(mappedData);
|
|
8725
8810
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8726
8811
|
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
8727
8812
|
}
|
|
8728
8813
|
return { mappedData, body: "" };
|
|
8729
8814
|
}
|
|
8730
|
-
async function handleUpdate$
|
|
8815
|
+
async function handleUpdate$4({ resource, id, data, mappedData }) {
|
|
8731
8816
|
const totalSize = calculateTotalSize(mappedData);
|
|
8732
8817
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8733
8818
|
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
8734
8819
|
}
|
|
8735
8820
|
return { mappedData, body: "" };
|
|
8736
8821
|
}
|
|
8737
|
-
async function handleUpsert$
|
|
8822
|
+
async function handleUpsert$4({ resource, id, data, mappedData }) {
|
|
8738
8823
|
const totalSize = calculateTotalSize(mappedData);
|
|
8739
8824
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8740
8825
|
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
8741
8826
|
}
|
|
8742
8827
|
return { mappedData, body: "" };
|
|
8743
8828
|
}
|
|
8744
|
-
async function handleGet$
|
|
8829
|
+
async function handleGet$4({ resource, metadata, body }) {
|
|
8745
8830
|
return { metadata, body };
|
|
8746
8831
|
}
|
|
8747
8832
|
|
|
8748
8833
|
var enforceLimits = /*#__PURE__*/Object.freeze({
|
|
8749
8834
|
__proto__: null,
|
|
8750
8835
|
S3_METADATA_LIMIT_BYTES: S3_METADATA_LIMIT_BYTES,
|
|
8751
|
-
handleGet: handleGet$
|
|
8752
|
-
handleInsert: handleInsert$
|
|
8753
|
-
handleUpdate: handleUpdate$
|
|
8754
|
-
handleUpsert: handleUpsert$
|
|
8836
|
+
handleGet: handleGet$4,
|
|
8837
|
+
handleInsert: handleInsert$4,
|
|
8838
|
+
handleUpdate: handleUpdate$4,
|
|
8839
|
+
handleUpsert: handleUpsert$4
|
|
8755
8840
|
});
|
|
8756
8841
|
|
|
8757
|
-
async function handleInsert$
|
|
8842
|
+
async function handleInsert$3({ resource, data, mappedData }) {
|
|
8758
8843
|
const totalSize = calculateTotalSize(mappedData);
|
|
8759
8844
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8760
8845
|
resource.emit("exceedsLimit", {
|
|
@@ -8767,7 +8852,7 @@ async function handleInsert$2({ resource, data, mappedData }) {
|
|
|
8767
8852
|
}
|
|
8768
8853
|
return { mappedData, body: "" };
|
|
8769
8854
|
}
|
|
8770
|
-
async function handleUpdate$
|
|
8855
|
+
async function handleUpdate$3({ resource, id, data, mappedData }) {
|
|
8771
8856
|
const totalSize = calculateTotalSize(mappedData);
|
|
8772
8857
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8773
8858
|
resource.emit("exceedsLimit", {
|
|
@@ -8781,7 +8866,7 @@ async function handleUpdate$2({ resource, id, data, mappedData }) {
|
|
|
8781
8866
|
}
|
|
8782
8867
|
return { mappedData, body: "" };
|
|
8783
8868
|
}
|
|
8784
|
-
async function handleUpsert$
|
|
8869
|
+
async function handleUpsert$3({ resource, id, data, mappedData }) {
|
|
8785
8870
|
const totalSize = calculateTotalSize(mappedData);
|
|
8786
8871
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8787
8872
|
resource.emit("exceedsLimit", {
|
|
@@ -8795,30 +8880,30 @@ async function handleUpsert$2({ resource, id, data, mappedData }) {
|
|
|
8795
8880
|
}
|
|
8796
8881
|
return { mappedData, body: "" };
|
|
8797
8882
|
}
|
|
8798
|
-
async function handleGet$
|
|
8883
|
+
async function handleGet$3({ resource, metadata, body }) {
|
|
8799
8884
|
return { metadata, body };
|
|
8800
8885
|
}
|
|
8801
8886
|
|
|
8802
|
-
var
|
|
8887
|
+
var userManaged = /*#__PURE__*/Object.freeze({
|
|
8803
8888
|
__proto__: null,
|
|
8804
|
-
handleGet: handleGet$
|
|
8805
|
-
handleInsert: handleInsert$
|
|
8806
|
-
handleUpdate: handleUpdate$
|
|
8807
|
-
handleUpsert: handleUpsert$
|
|
8889
|
+
handleGet: handleGet$3,
|
|
8890
|
+
handleInsert: handleInsert$3,
|
|
8891
|
+
handleUpdate: handleUpdate$3,
|
|
8892
|
+
handleUpsert: handleUpsert$3
|
|
8808
8893
|
});
|
|
8809
8894
|
|
|
8810
8895
|
const TRUNCATE_SUFFIX = "...";
|
|
8811
8896
|
const TRUNCATE_SUFFIX_BYTES = calculateUTF8Bytes(TRUNCATE_SUFFIX);
|
|
8812
|
-
async function handleInsert$
|
|
8897
|
+
async function handleInsert$2({ resource, data, mappedData }) {
|
|
8813
8898
|
return handleTruncate({ resource, data, mappedData });
|
|
8814
8899
|
}
|
|
8815
|
-
async function handleUpdate$
|
|
8900
|
+
async function handleUpdate$2({ resource, id, data, mappedData }) {
|
|
8816
8901
|
return handleTruncate({ resource, data, mappedData });
|
|
8817
8902
|
}
|
|
8818
|
-
async function handleUpsert$
|
|
8903
|
+
async function handleUpsert$2({ resource, id, data, mappedData }) {
|
|
8819
8904
|
return handleTruncate({ resource, data, mappedData });
|
|
8820
8905
|
}
|
|
8821
|
-
async function handleGet$
|
|
8906
|
+
async function handleGet$2({ resource, metadata, body }) {
|
|
8822
8907
|
return { metadata, body };
|
|
8823
8908
|
}
|
|
8824
8909
|
function handleTruncate({ resource, data, mappedData }) {
|
|
@@ -8858,25 +8943,25 @@ function handleTruncate({ resource, data, mappedData }) {
|
|
|
8858
8943
|
|
|
8859
8944
|
var dataTruncate = /*#__PURE__*/Object.freeze({
|
|
8860
8945
|
__proto__: null,
|
|
8861
|
-
handleGet: handleGet$
|
|
8862
|
-
handleInsert: handleInsert$
|
|
8863
|
-
handleUpdate: handleUpdate$
|
|
8864
|
-
handleUpsert: handleUpsert$
|
|
8946
|
+
handleGet: handleGet$2,
|
|
8947
|
+
handleInsert: handleInsert$2,
|
|
8948
|
+
handleUpdate: handleUpdate$2,
|
|
8949
|
+
handleUpsert: handleUpsert$2
|
|
8865
8950
|
});
|
|
8866
8951
|
|
|
8867
8952
|
const OVERFLOW_FLAG = "$overflow";
|
|
8868
8953
|
const OVERFLOW_FLAG_VALUE = "true";
|
|
8869
8954
|
const OVERFLOW_FLAG_BYTES = calculateUTF8Bytes(OVERFLOW_FLAG) + calculateUTF8Bytes(OVERFLOW_FLAG_VALUE);
|
|
8870
|
-
async function handleInsert({ resource, data, mappedData }) {
|
|
8955
|
+
async function handleInsert$1({ resource, data, mappedData }) {
|
|
8871
8956
|
return handleOverflow({ resource, data, mappedData });
|
|
8872
8957
|
}
|
|
8873
|
-
async function handleUpdate({ resource, id, data, mappedData }) {
|
|
8958
|
+
async function handleUpdate$1({ resource, id, data, mappedData }) {
|
|
8874
8959
|
return handleOverflow({ resource, data, mappedData });
|
|
8875
8960
|
}
|
|
8876
|
-
async function handleUpsert({ resource, id, data, mappedData }) {
|
|
8961
|
+
async function handleUpsert$1({ resource, id, data, mappedData }) {
|
|
8877
8962
|
return handleOverflow({ resource, data, mappedData });
|
|
8878
8963
|
}
|
|
8879
|
-
async function handleGet({ resource, metadata, body }) {
|
|
8964
|
+
async function handleGet$1({ resource, metadata, body }) {
|
|
8880
8965
|
if (metadata[OVERFLOW_FLAG] === OVERFLOW_FLAG_VALUE) {
|
|
8881
8966
|
try {
|
|
8882
8967
|
const bodyData = body ? JSON.parse(body) : {};
|
|
@@ -8918,6 +9003,51 @@ function handleOverflow({ resource, data, mappedData }) {
|
|
|
8918
9003
|
}
|
|
8919
9004
|
|
|
8920
9005
|
var bodyOverflow = /*#__PURE__*/Object.freeze({
|
|
9006
|
+
__proto__: null,
|
|
9007
|
+
handleGet: handleGet$1,
|
|
9008
|
+
handleInsert: handleInsert$1,
|
|
9009
|
+
handleUpdate: handleUpdate$1,
|
|
9010
|
+
handleUpsert: handleUpsert$1
|
|
9011
|
+
});
|
|
9012
|
+
|
|
9013
|
+
async function handleInsert({ resource, data, mappedData }) {
|
|
9014
|
+
const bodyContent = JSON.stringify(mappedData);
|
|
9015
|
+
return {
|
|
9016
|
+
mappedData: {},
|
|
9017
|
+
body: bodyContent
|
|
9018
|
+
};
|
|
9019
|
+
}
|
|
9020
|
+
async function handleUpdate({ resource, id, data, mappedData }) {
|
|
9021
|
+
const bodyContent = JSON.stringify(mappedData);
|
|
9022
|
+
return {
|
|
9023
|
+
mappedData: {},
|
|
9024
|
+
body: bodyContent
|
|
9025
|
+
};
|
|
9026
|
+
}
|
|
9027
|
+
async function handleUpsert({ resource, id, data, mappedData }) {
|
|
9028
|
+
const bodyContent = JSON.stringify(mappedData);
|
|
9029
|
+
return {
|
|
9030
|
+
mappedData: {},
|
|
9031
|
+
body: bodyContent
|
|
9032
|
+
};
|
|
9033
|
+
}
|
|
9034
|
+
async function handleGet({ resource, metadata, body }) {
|
|
9035
|
+
try {
|
|
9036
|
+
const bodyData = body ? JSON.parse(body) : {};
|
|
9037
|
+
return {
|
|
9038
|
+
metadata: bodyData,
|
|
9039
|
+
body: ""
|
|
9040
|
+
};
|
|
9041
|
+
} catch (error) {
|
|
9042
|
+
console.warn(`Failed to parse body-only content:`, error.message);
|
|
9043
|
+
return {
|
|
9044
|
+
metadata,
|
|
9045
|
+
body: ""
|
|
9046
|
+
};
|
|
9047
|
+
}
|
|
9048
|
+
}
|
|
9049
|
+
|
|
9050
|
+
var bodyOnly = /*#__PURE__*/Object.freeze({
|
|
8921
9051
|
__proto__: null,
|
|
8922
9052
|
handleGet: handleGet,
|
|
8923
9053
|
handleInsert: handleInsert,
|
|
@@ -8926,10 +9056,11 @@ var bodyOverflow = /*#__PURE__*/Object.freeze({
|
|
|
8926
9056
|
});
|
|
8927
9057
|
|
|
8928
9058
|
const behaviors = {
|
|
8929
|
-
"user-
|
|
9059
|
+
"user-managed": userManaged,
|
|
8930
9060
|
"enforce-limits": enforceLimits,
|
|
8931
9061
|
"data-truncate": dataTruncate,
|
|
8932
|
-
"body-overflow": bodyOverflow
|
|
9062
|
+
"body-overflow": bodyOverflow,
|
|
9063
|
+
"body-only": bodyOnly
|
|
8933
9064
|
};
|
|
8934
9065
|
function getBehavior(behaviorName) {
|
|
8935
9066
|
const behavior = behaviors[behaviorName];
|
|
@@ -8939,7 +9070,7 @@ function getBehavior(behaviorName) {
|
|
|
8939
9070
|
return behavior;
|
|
8940
9071
|
}
|
|
8941
9072
|
const AVAILABLE_BEHAVIORS = Object.keys(behaviors);
|
|
8942
|
-
const DEFAULT_BEHAVIOR = "user-
|
|
9073
|
+
const DEFAULT_BEHAVIOR = "user-managed";
|
|
8943
9074
|
|
|
8944
9075
|
function createIdGeneratorWithSize(size) {
|
|
8945
9076
|
return nanoid.customAlphabet(nanoid.urlAlphabet, size);
|
|
@@ -8952,7 +9083,7 @@ class Resource extends EventEmitter {
|
|
|
8952
9083
|
* @param {Object} config.client - S3 client instance
|
|
8953
9084
|
* @param {string} [config.version='v0'] - Resource version
|
|
8954
9085
|
* @param {Object} [config.attributes={}] - Resource attributes schema
|
|
8955
|
-
* @param {string} [config.behavior='user-
|
|
9086
|
+
* @param {string} [config.behavior='user-managed'] - Resource behavior strategy
|
|
8956
9087
|
* @param {string} [config.passphrase='secret'] - Encryption passphrase
|
|
8957
9088
|
* @param {number} [config.parallelism=10] - Parallelism for bulk operations
|
|
8958
9089
|
* @param {Array} [config.observers=[]] - Observer instances
|
|
@@ -8966,6 +9097,7 @@ class Resource extends EventEmitter {
|
|
|
8966
9097
|
* @param {Object} [config.options={}] - Additional options
|
|
8967
9098
|
* @param {Function} [config.idGenerator] - Custom ID generator function
|
|
8968
9099
|
* @param {number} [config.idSize=22] - Size for auto-generated IDs
|
|
9100
|
+
* @param {boolean} [config.versioningEnabled=false] - Enable versioning for this resource
|
|
8969
9101
|
* @example
|
|
8970
9102
|
* const users = new Resource({
|
|
8971
9103
|
* name: 'users',
|
|
@@ -8975,7 +9107,7 @@ class Resource extends EventEmitter {
|
|
|
8975
9107
|
* email: 'string|required',
|
|
8976
9108
|
* password: 'secret|required'
|
|
8977
9109
|
* },
|
|
8978
|
-
* behavior: 'user-
|
|
9110
|
+
* behavior: 'user-managed',
|
|
8979
9111
|
* passphrase: 'my-secret-key',
|
|
8980
9112
|
* timestamps: true,
|
|
8981
9113
|
* partitions: {
|
|
@@ -9039,7 +9171,8 @@ ${validation.errors.join("\n")}`);
|
|
|
9039
9171
|
allNestedObjectsOptional = true,
|
|
9040
9172
|
hooks = {},
|
|
9041
9173
|
idGenerator: customIdGenerator,
|
|
9042
|
-
idSize = 22
|
|
9174
|
+
idSize = 22,
|
|
9175
|
+
versioningEnabled = false
|
|
9043
9176
|
} = config;
|
|
9044
9177
|
this.name = name;
|
|
9045
9178
|
this.client = client;
|
|
@@ -9048,6 +9181,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9048
9181
|
this.observers = observers;
|
|
9049
9182
|
this.parallelism = parallelism;
|
|
9050
9183
|
this.passphrase = passphrase ?? "secret";
|
|
9184
|
+
this.versioningEnabled = versioningEnabled;
|
|
9051
9185
|
this.idGenerator = this.configureIdGenerator(customIdGenerator, idSize);
|
|
9052
9186
|
this.config = {
|
|
9053
9187
|
cache,
|
|
@@ -9155,6 +9289,15 @@ ${validation.errors.join("\n")}`);
|
|
|
9155
9289
|
}
|
|
9156
9290
|
}
|
|
9157
9291
|
this.setupPartitionHooks();
|
|
9292
|
+
if (this.versioningEnabled) {
|
|
9293
|
+
if (!this.config.partitions.byVersion) {
|
|
9294
|
+
this.config.partitions.byVersion = {
|
|
9295
|
+
fields: {
|
|
9296
|
+
_v: "string"
|
|
9297
|
+
}
|
|
9298
|
+
};
|
|
9299
|
+
}
|
|
9300
|
+
}
|
|
9158
9301
|
this.schema = new Schema({
|
|
9159
9302
|
name: this.name,
|
|
9160
9303
|
attributes: this.attributes,
|
|
@@ -9274,6 +9417,9 @@ ${validation.errors.join("\n")}`);
|
|
|
9274
9417
|
* @returns {boolean} True if field exists
|
|
9275
9418
|
*/
|
|
9276
9419
|
fieldExistsInAttributes(fieldName) {
|
|
9420
|
+
if (fieldName.startsWith("_")) {
|
|
9421
|
+
return true;
|
|
9422
|
+
}
|
|
9277
9423
|
if (!fieldName.includes(".")) {
|
|
9278
9424
|
return Object.keys(this.attributes || {}).includes(fieldName);
|
|
9279
9425
|
}
|
|
@@ -9327,12 +9473,12 @@ ${validation.errors.join("\n")}`);
|
|
|
9327
9473
|
return transformedValue;
|
|
9328
9474
|
}
|
|
9329
9475
|
/**
|
|
9330
|
-
* Get the main resource key (
|
|
9476
|
+
* Get the main resource key (new format without version in path)
|
|
9331
9477
|
* @param {string} id - Resource ID
|
|
9332
9478
|
* @returns {string} The main S3 key path
|
|
9333
9479
|
*/
|
|
9334
9480
|
getResourceKey(id) {
|
|
9335
|
-
return join(`resource=${this.name}`, `
|
|
9481
|
+
return join(`resource=${this.name}`, `data`, `id=${id}`);
|
|
9336
9482
|
}
|
|
9337
9483
|
/**
|
|
9338
9484
|
* Generate partition key for a resource in a specific partition
|
|
@@ -9401,12 +9547,23 @@ ${validation.errors.join("\n")}`);
|
|
|
9401
9547
|
}
|
|
9402
9548
|
return currentLevel;
|
|
9403
9549
|
}
|
|
9550
|
+
/**
|
|
9551
|
+
* Calculate estimated content length for body data
|
|
9552
|
+
* @param {string|Buffer} body - Body content
|
|
9553
|
+
* @returns {number} Estimated content length in bytes
|
|
9554
|
+
*/
|
|
9555
|
+
calculateContentLength(body) {
|
|
9556
|
+
if (!body) return 0;
|
|
9557
|
+
if (Buffer.isBuffer(body)) return body.length;
|
|
9558
|
+
if (typeof body === "string") return Buffer.byteLength(body, "utf8");
|
|
9559
|
+
if (typeof body === "object") return Buffer.byteLength(JSON.stringify(body), "utf8");
|
|
9560
|
+
return Buffer.byteLength(String(body), "utf8");
|
|
9561
|
+
}
|
|
9404
9562
|
/**
|
|
9405
9563
|
* Insert a new resource object
|
|
9406
|
-
* @param {Object}
|
|
9407
|
-
* @param {string} [
|
|
9408
|
-
* @
|
|
9409
|
-
* @returns {Promise<Object>} The inserted resource object with all attributes and generated ID
|
|
9564
|
+
* @param {Object} attributes - Resource attributes
|
|
9565
|
+
* @param {string} [attributes.id] - Custom ID (optional, auto-generated if not provided)
|
|
9566
|
+
* @returns {Promise<Object>} The created resource object with all attributes
|
|
9410
9567
|
* @example
|
|
9411
9568
|
* // Insert with auto-generated ID
|
|
9412
9569
|
* const user = await resource.insert({
|
|
@@ -9414,18 +9571,13 @@ ${validation.errors.join("\n")}`);
|
|
|
9414
9571
|
* email: 'john@example.com',
|
|
9415
9572
|
* age: 30
|
|
9416
9573
|
* });
|
|
9574
|
+
* console.log(user.id); // Auto-generated ID
|
|
9417
9575
|
*
|
|
9418
9576
|
* // Insert with custom ID
|
|
9419
9577
|
* const user = await resource.insert({
|
|
9420
|
-
* id: '
|
|
9421
|
-
* name: 'Jane Smith',
|
|
9422
|
-
* email: 'jane@example.com'
|
|
9423
|
-
* });
|
|
9424
|
-
*
|
|
9425
|
-
* // Insert with auto-generated password for secret field
|
|
9426
|
-
* const user = await resource.insert({
|
|
9578
|
+
* id: 'user-123',
|
|
9427
9579
|
* name: 'John Doe',
|
|
9428
|
-
* email: 'john@example.com'
|
|
9580
|
+
* email: 'john@example.com'
|
|
9429
9581
|
* });
|
|
9430
9582
|
*/
|
|
9431
9583
|
async insert({ id, ...attributes }) {
|
|
@@ -9433,7 +9585,8 @@ ${validation.errors.join("\n")}`);
|
|
|
9433
9585
|
attributes.createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9434
9586
|
attributes.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9435
9587
|
}
|
|
9436
|
-
const
|
|
9588
|
+
const completeData = { id, ...attributes };
|
|
9589
|
+
const preProcessedData = await this.executeHooks("preInsert", completeData);
|
|
9437
9590
|
const {
|
|
9438
9591
|
errors,
|
|
9439
9592
|
isValid,
|
|
@@ -9447,15 +9600,18 @@ ${validation.errors.join("\n")}`);
|
|
|
9447
9600
|
validation: errors
|
|
9448
9601
|
});
|
|
9449
9602
|
}
|
|
9450
|
-
|
|
9451
|
-
const
|
|
9603
|
+
const { id: validatedId, ...validatedAttributes } = validated;
|
|
9604
|
+
const finalId = validatedId || id || this.idGenerator();
|
|
9605
|
+
const mappedData = await this.schema.mapper(validatedAttributes);
|
|
9606
|
+
mappedData._v = String(this.version);
|
|
9452
9607
|
const behaviorImpl = getBehavior(this.behavior);
|
|
9453
9608
|
const { mappedData: processedMetadata, body } = await behaviorImpl.handleInsert({
|
|
9454
9609
|
resource: this,
|
|
9455
|
-
data:
|
|
9610
|
+
data: validatedAttributes,
|
|
9456
9611
|
mappedData
|
|
9457
9612
|
});
|
|
9458
|
-
const
|
|
9613
|
+
const finalMetadata = processedMetadata;
|
|
9614
|
+
const key = this.getResourceKey(finalId);
|
|
9459
9615
|
let contentType = void 0;
|
|
9460
9616
|
if (body && body !== "") {
|
|
9461
9617
|
try {
|
|
@@ -9465,12 +9621,13 @@ ${validation.errors.join("\n")}`);
|
|
|
9465
9621
|
}
|
|
9466
9622
|
}
|
|
9467
9623
|
await this.client.putObject({
|
|
9468
|
-
metadata:
|
|
9624
|
+
metadata: finalMetadata,
|
|
9469
9625
|
key,
|
|
9470
9626
|
body,
|
|
9471
|
-
contentType
|
|
9627
|
+
contentType,
|
|
9628
|
+
contentLength: this.calculateContentLength(body)
|
|
9472
9629
|
});
|
|
9473
|
-
const final = lodashEs.merge({ id },
|
|
9630
|
+
const final = lodashEs.merge({ id: finalId }, validatedAttributes);
|
|
9474
9631
|
await this.executeHooks("afterInsert", final);
|
|
9475
9632
|
this.emit("insert", final);
|
|
9476
9633
|
return final;
|
|
@@ -9489,7 +9646,8 @@ ${validation.errors.join("\n")}`);
|
|
|
9489
9646
|
const key = this.getResourceKey(id);
|
|
9490
9647
|
try {
|
|
9491
9648
|
const request = await this.client.headObject(key);
|
|
9492
|
-
const
|
|
9649
|
+
const objectVersionRaw = request.Metadata?._v || this.version;
|
|
9650
|
+
const objectVersion = typeof objectVersionRaw === "string" && objectVersionRaw.startsWith("v") ? objectVersionRaw.slice(1) : objectVersionRaw;
|
|
9493
9651
|
const schema = await this.getSchemaForVersion(objectVersion);
|
|
9494
9652
|
let metadata = await schema.unmapper(request.Metadata);
|
|
9495
9653
|
const behaviorImpl = getBehavior(this.behavior);
|
|
@@ -9514,9 +9672,13 @@ ${validation.errors.join("\n")}`);
|
|
|
9514
9672
|
data._lastModified = request.LastModified;
|
|
9515
9673
|
data._hasContent = request.ContentLength > 0;
|
|
9516
9674
|
data._mimeType = request.ContentType || null;
|
|
9675
|
+
data._v = objectVersion;
|
|
9517
9676
|
if (request.VersionId) data._versionId = request.VersionId;
|
|
9518
9677
|
if (request.Expiration) data._expiresAt = request.Expiration;
|
|
9519
9678
|
data._definitionHash = this.getDefinitionHash();
|
|
9679
|
+
if (objectVersion !== this.version) {
|
|
9680
|
+
data = await this.applyVersionMapping(data, objectVersion, this.version);
|
|
9681
|
+
}
|
|
9520
9682
|
this.emit("get", data);
|
|
9521
9683
|
return data;
|
|
9522
9684
|
} catch (error) {
|
|
@@ -9622,28 +9784,30 @@ ${validation.errors.join("\n")}`);
|
|
|
9622
9784
|
attributes.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9623
9785
|
}
|
|
9624
9786
|
const preProcessedData = await this.executeHooks("preUpdate", attributes);
|
|
9625
|
-
const
|
|
9626
|
-
|
|
9627
|
-
const { isValid, errors, data: validated } = await this.validate(attrs);
|
|
9787
|
+
const completeData = { ...originalData, ...preProcessedData, id };
|
|
9788
|
+
const { isValid, errors, data: validated } = await this.validate(completeData);
|
|
9628
9789
|
if (!isValid) {
|
|
9629
9790
|
throw new InvalidResourceItem({
|
|
9630
|
-
bucket: this.client.bucket,
|
|
9791
|
+
bucket: this.client.config.bucket,
|
|
9631
9792
|
resourceName: this.name,
|
|
9632
9793
|
attributes: preProcessedData,
|
|
9633
9794
|
validation: errors
|
|
9634
9795
|
});
|
|
9635
9796
|
}
|
|
9797
|
+
const { id: validatedId, ...validatedAttributes } = validated;
|
|
9636
9798
|
const oldData = { ...originalData, id };
|
|
9637
|
-
const newData = { ...
|
|
9799
|
+
const newData = { ...validatedAttributes, id };
|
|
9638
9800
|
await this.handlePartitionReferenceUpdates(oldData, newData);
|
|
9639
|
-
const mappedData = await this.schema.mapper(
|
|
9801
|
+
const mappedData = await this.schema.mapper(validatedAttributes);
|
|
9802
|
+
mappedData._v = String(this.version);
|
|
9640
9803
|
const behaviorImpl = getBehavior(this.behavior);
|
|
9641
9804
|
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
9642
9805
|
resource: this,
|
|
9643
9806
|
id,
|
|
9644
|
-
data:
|
|
9807
|
+
data: validatedAttributes,
|
|
9645
9808
|
mappedData
|
|
9646
9809
|
});
|
|
9810
|
+
const finalMetadata = processedMetadata;
|
|
9647
9811
|
const key = this.getResourceKey(id);
|
|
9648
9812
|
let existingContentType = void 0;
|
|
9649
9813
|
let finalBody = body;
|
|
@@ -9671,16 +9835,19 @@ ${validation.errors.join("\n")}`);
|
|
|
9671
9835
|
} catch {
|
|
9672
9836
|
}
|
|
9673
9837
|
}
|
|
9838
|
+
if (this.versioningEnabled && originalData._v !== this.version) {
|
|
9839
|
+
await this.createHistoricalVersion(id, originalData);
|
|
9840
|
+
}
|
|
9674
9841
|
await this.client.putObject({
|
|
9675
9842
|
key,
|
|
9676
9843
|
body: finalBody,
|
|
9677
9844
|
contentType: finalContentType,
|
|
9678
|
-
metadata:
|
|
9845
|
+
metadata: finalMetadata
|
|
9679
9846
|
});
|
|
9680
|
-
|
|
9681
|
-
await this.executeHooks("afterUpdate",
|
|
9682
|
-
this.emit("update", preProcessedData,
|
|
9683
|
-
return
|
|
9847
|
+
validatedAttributes.id = id;
|
|
9848
|
+
await this.executeHooks("afterUpdate", validatedAttributes);
|
|
9849
|
+
this.emit("update", preProcessedData, validatedAttributes);
|
|
9850
|
+
return validatedAttributes;
|
|
9684
9851
|
}
|
|
9685
9852
|
/**
|
|
9686
9853
|
* Delete a resource object by ID
|
|
@@ -9769,7 +9936,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9769
9936
|
prefix = `resource=${this.name}/partition=${partition}`;
|
|
9770
9937
|
}
|
|
9771
9938
|
} else {
|
|
9772
|
-
prefix = `resource=${this.name}/
|
|
9939
|
+
prefix = `resource=${this.name}/data`;
|
|
9773
9940
|
}
|
|
9774
9941
|
const count = await this.client.count({ prefix });
|
|
9775
9942
|
this.emit("count", count);
|
|
@@ -9838,7 +10005,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9838
10005
|
`deleteAll() is a dangerous operation and requires paranoid: false option. Current paranoid setting: ${this.config.paranoid}`
|
|
9839
10006
|
);
|
|
9840
10007
|
}
|
|
9841
|
-
const prefix = `resource=${this.name}/
|
|
10008
|
+
const prefix = `resource=${this.name}/data`;
|
|
9842
10009
|
const deletedCount = await this.client.deleteAll({ prefix });
|
|
9843
10010
|
this.emit("deleteAll", {
|
|
9844
10011
|
version: this.version,
|
|
@@ -9916,7 +10083,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9916
10083
|
prefix = `resource=${this.name}/partition=${partition}`;
|
|
9917
10084
|
}
|
|
9918
10085
|
} else {
|
|
9919
|
-
prefix = `resource=${this.name}/
|
|
10086
|
+
prefix = `resource=${this.name}/data`;
|
|
9920
10087
|
}
|
|
9921
10088
|
const keys = await this.client.getKeysPage({
|
|
9922
10089
|
prefix,
|
|
@@ -9933,154 +10100,175 @@ ${validation.errors.join("\n")}`);
|
|
|
9933
10100
|
return ids;
|
|
9934
10101
|
}
|
|
9935
10102
|
/**
|
|
9936
|
-
* List
|
|
10103
|
+
* List resources with optional partition filtering and pagination
|
|
9937
10104
|
* @param {Object} [params] - List parameters
|
|
9938
10105
|
* @param {string} [params.partition] - Partition name to list from
|
|
9939
10106
|
* @param {Object} [params.partitionValues] - Partition field values to filter by
|
|
9940
|
-
* @param {number} [params.limit] - Maximum number of results
|
|
9941
|
-
* @param {number} [params.offset=0] -
|
|
9942
|
-
* @returns {Promise<Object[]>} Array of resource objects
|
|
10107
|
+
* @param {number} [params.limit] - Maximum number of results
|
|
10108
|
+
* @param {number} [params.offset=0] - Number of results to skip
|
|
10109
|
+
* @returns {Promise<Object[]>} Array of resource objects
|
|
9943
10110
|
* @example
|
|
9944
10111
|
* // List all resources
|
|
9945
10112
|
* const allUsers = await resource.list();
|
|
9946
10113
|
*
|
|
9947
10114
|
* // List with pagination
|
|
9948
|
-
* const
|
|
9949
|
-
* const secondPage = await resource.list({ limit: 10, offset: 10 });
|
|
10115
|
+
* const first10 = await resource.list({ limit: 10, offset: 0 });
|
|
9950
10116
|
*
|
|
9951
10117
|
* // List from specific partition
|
|
9952
|
-
* const
|
|
9953
|
-
* partition: '
|
|
9954
|
-
* partitionValues: { '
|
|
9955
|
-
* });
|
|
9956
|
-
*
|
|
9957
|
-
* // List from partition with pagination
|
|
9958
|
-
* const googleUsersPage = await resource.list({
|
|
9959
|
-
* partition: 'byUtmSource',
|
|
9960
|
-
* partitionValues: { 'utm.source': 'google' },
|
|
9961
|
-
* limit: 5,
|
|
9962
|
-
* offset: 0
|
|
10118
|
+
* const usUsers = await resource.list({
|
|
10119
|
+
* partition: 'byCountry',
|
|
10120
|
+
* partitionValues: { 'profile.country': 'US' }
|
|
9963
10121
|
* });
|
|
9964
10122
|
*/
|
|
9965
10123
|
async list({ partition = null, partitionValues = {}, limit, offset = 0 } = {}) {
|
|
9966
10124
|
try {
|
|
9967
10125
|
if (!partition) {
|
|
9968
|
-
|
|
9969
|
-
try {
|
|
9970
|
-
ids2 = await this.listIds({ partition, partitionValues });
|
|
9971
|
-
} catch (listIdsError) {
|
|
9972
|
-
console.warn(`Failed to get list IDs:`, listIdsError.message);
|
|
9973
|
-
return [];
|
|
9974
|
-
}
|
|
9975
|
-
let filteredIds2 = ids2.slice(offset);
|
|
9976
|
-
if (limit) {
|
|
9977
|
-
filteredIds2 = filteredIds2.slice(0, limit);
|
|
9978
|
-
}
|
|
9979
|
-
const { results: results2, errors: errors2 } = await promisePool.PromisePool.for(filteredIds2).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
9980
|
-
console.warn(`Failed to get resource ${id}:`, error.message);
|
|
9981
|
-
return null;
|
|
9982
|
-
}).process(async (id) => {
|
|
9983
|
-
try {
|
|
9984
|
-
return await this.get(id);
|
|
9985
|
-
} catch (error) {
|
|
9986
|
-
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
9987
|
-
console.warn(`Decryption failed for ${id}, returning basic info`);
|
|
9988
|
-
return {
|
|
9989
|
-
id,
|
|
9990
|
-
_decryptionFailed: true,
|
|
9991
|
-
_error: error.message
|
|
9992
|
-
};
|
|
9993
|
-
}
|
|
9994
|
-
throw error;
|
|
9995
|
-
}
|
|
9996
|
-
});
|
|
9997
|
-
const validResults2 = results2.filter((item) => item !== null);
|
|
9998
|
-
this.emit("list", { partition, partitionValues, count: validResults2.length, errors: errors2.length });
|
|
9999
|
-
return validResults2;
|
|
10000
|
-
}
|
|
10001
|
-
if (!this.config.partitions || !this.config.partitions[partition]) {
|
|
10002
|
-
console.warn(`Partition '${partition}' not found in resource '${this.name}'`);
|
|
10003
|
-
this.emit("list", { partition, partitionValues, count: 0, errors: 0 });
|
|
10004
|
-
return [];
|
|
10126
|
+
return await this.listMain({ limit, offset });
|
|
10005
10127
|
}
|
|
10006
|
-
|
|
10007
|
-
|
|
10008
|
-
|
|
10009
|
-
|
|
10010
|
-
|
|
10011
|
-
|
|
10012
|
-
|
|
10013
|
-
|
|
10014
|
-
|
|
10128
|
+
return await this.listPartition({ partition, partitionValues, limit, offset });
|
|
10129
|
+
} catch (error) {
|
|
10130
|
+
return this.handleListError(error, { partition, partitionValues });
|
|
10131
|
+
}
|
|
10132
|
+
}
|
|
10133
|
+
/**
|
|
10134
|
+
* List resources from main resource (no partition)
|
|
10135
|
+
*/
|
|
10136
|
+
async listMain({ limit, offset = 0 }) {
|
|
10137
|
+
const ids = await this.listIds({ limit, offset });
|
|
10138
|
+
const results = await this.processListResults(ids, "main");
|
|
10139
|
+
this.emit("list", { count: results.length, errors: 0 });
|
|
10140
|
+
return results;
|
|
10141
|
+
}
|
|
10142
|
+
/**
|
|
10143
|
+
* List resources from specific partition
|
|
10144
|
+
*/
|
|
10145
|
+
async listPartition({ partition, partitionValues, limit, offset = 0 }) {
|
|
10146
|
+
if (!this.config.partitions?.[partition]) {
|
|
10147
|
+
console.warn(`Partition '${partition}' not found in resource '${this.name}'`);
|
|
10148
|
+
this.emit("list", { partition, partitionValues, count: 0, errors: 0 });
|
|
10149
|
+
return [];
|
|
10150
|
+
}
|
|
10151
|
+
const partitionDef = this.config.partitions[partition];
|
|
10152
|
+
const prefix = this.buildPartitionPrefix(partition, partitionDef, partitionValues);
|
|
10153
|
+
const keys = await this.client.getAllKeys({ prefix });
|
|
10154
|
+
const ids = this.extractIdsFromKeys(keys).slice(offset);
|
|
10155
|
+
const filteredIds = limit ? ids.slice(0, limit) : ids;
|
|
10156
|
+
const results = await this.processPartitionResults(filteredIds, partition, partitionDef, keys);
|
|
10157
|
+
this.emit("list", { partition, partitionValues, count: results.length, errors: 0 });
|
|
10158
|
+
return results;
|
|
10159
|
+
}
|
|
10160
|
+
/**
|
|
10161
|
+
* Build partition prefix from partition definition and values
|
|
10162
|
+
*/
|
|
10163
|
+
buildPartitionPrefix(partition, partitionDef, partitionValues) {
|
|
10164
|
+
const partitionSegments = [];
|
|
10165
|
+
const sortedFields = Object.entries(partitionDef.fields).sort(([a], [b]) => a.localeCompare(b));
|
|
10166
|
+
for (const [fieldName, rule] of sortedFields) {
|
|
10167
|
+
const value = partitionValues[fieldName];
|
|
10168
|
+
if (value !== void 0 && value !== null) {
|
|
10169
|
+
const transformedValue = this.applyPartitionRule(value, rule);
|
|
10170
|
+
partitionSegments.push(`${fieldName}=${transformedValue}`);
|
|
10015
10171
|
}
|
|
10016
|
-
|
|
10017
|
-
|
|
10018
|
-
|
|
10019
|
-
|
|
10020
|
-
|
|
10172
|
+
}
|
|
10173
|
+
if (partitionSegments.length > 0) {
|
|
10174
|
+
return `resource=${this.name}/partition=${partition}/${partitionSegments.join("/")}`;
|
|
10175
|
+
}
|
|
10176
|
+
return `resource=${this.name}/partition=${partition}`;
|
|
10177
|
+
}
|
|
10178
|
+
/**
|
|
10179
|
+
* Extract IDs from S3 keys
|
|
10180
|
+
*/
|
|
10181
|
+
extractIdsFromKeys(keys) {
|
|
10182
|
+
return keys.map((key) => {
|
|
10183
|
+
const parts = key.split("/");
|
|
10184
|
+
const idPart = parts.find((part) => part.startsWith("id="));
|
|
10185
|
+
return idPart ? idPart.replace("id=", "") : null;
|
|
10186
|
+
}).filter(Boolean);
|
|
10187
|
+
}
|
|
10188
|
+
/**
|
|
10189
|
+
* Process list results with error handling
|
|
10190
|
+
*/
|
|
10191
|
+
async processListResults(ids, context = "main") {
|
|
10192
|
+
const { results, errors } = await promisePool.PromisePool.for(ids).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
10193
|
+
console.warn(`Failed to get ${context} resource ${id}:`, error.message);
|
|
10194
|
+
return null;
|
|
10195
|
+
}).process(async (id) => {
|
|
10196
|
+
try {
|
|
10197
|
+
return await this.get(id);
|
|
10198
|
+
} catch (error) {
|
|
10199
|
+
return this.handleResourceError(error, id, context);
|
|
10021
10200
|
}
|
|
10022
|
-
|
|
10201
|
+
});
|
|
10202
|
+
return results.filter((item) => item !== null);
|
|
10203
|
+
}
|
|
10204
|
+
/**
|
|
10205
|
+
* Process partition results with error handling
|
|
10206
|
+
*/
|
|
10207
|
+
async processPartitionResults(ids, partition, partitionDef, keys) {
|
|
10208
|
+
const sortedFields = Object.entries(partitionDef.fields).sort(([a], [b]) => a.localeCompare(b));
|
|
10209
|
+
const { results, errors } = await promisePool.PromisePool.for(ids).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
10210
|
+
console.warn(`Failed to get partition resource ${id}:`, error.message);
|
|
10211
|
+
return null;
|
|
10212
|
+
}).process(async (id) => {
|
|
10023
10213
|
try {
|
|
10024
|
-
|
|
10025
|
-
|
|
10026
|
-
|
|
10027
|
-
|
|
10214
|
+
const actualPartitionValues = this.extractPartitionValuesFromKey(id, keys, sortedFields);
|
|
10215
|
+
return await this.getFromPartition({
|
|
10216
|
+
id,
|
|
10217
|
+
partitionName: partition,
|
|
10218
|
+
partitionValues: actualPartitionValues
|
|
10219
|
+
});
|
|
10220
|
+
} catch (error) {
|
|
10221
|
+
return this.handleResourceError(error, id, "partition");
|
|
10028
10222
|
}
|
|
10029
|
-
|
|
10030
|
-
|
|
10031
|
-
|
|
10032
|
-
|
|
10033
|
-
|
|
10034
|
-
|
|
10035
|
-
|
|
10036
|
-
|
|
10037
|
-
|
|
10038
|
-
|
|
10039
|
-
|
|
10040
|
-
|
|
10041
|
-
|
|
10042
|
-
|
|
10043
|
-
|
|
10044
|
-
|
|
10045
|
-
|
|
10046
|
-
|
|
10047
|
-
const keyParts = keyForId.split("/");
|
|
10048
|
-
const actualPartitionValues = {};
|
|
10049
|
-
for (const [fieldName, rule] of sortedFields) {
|
|
10050
|
-
const fieldPart = keyParts.find((part) => part.startsWith(`${fieldName}=`));
|
|
10051
|
-
if (fieldPart) {
|
|
10052
|
-
const value = fieldPart.replace(`${fieldName}=`, "");
|
|
10053
|
-
actualPartitionValues[fieldName] = value;
|
|
10054
|
-
}
|
|
10055
|
-
}
|
|
10056
|
-
return await this.getFromPartition({ id, partitionName: partition, partitionValues: actualPartitionValues });
|
|
10057
|
-
} catch (error) {
|
|
10058
|
-
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
10059
|
-
console.warn(`Decryption failed for partition resource ${id}, returning basic info`);
|
|
10060
|
-
return {
|
|
10061
|
-
id,
|
|
10062
|
-
_partition: partition,
|
|
10063
|
-
_partitionValues: partitionValues,
|
|
10064
|
-
_decryptionFailed: true,
|
|
10065
|
-
_error: error.message
|
|
10066
|
-
};
|
|
10067
|
-
}
|
|
10068
|
-
throw error;
|
|
10069
|
-
}
|
|
10070
|
-
});
|
|
10071
|
-
const validResults = results.filter((item) => item !== null);
|
|
10072
|
-
this.emit("list", { partition, partitionValues, count: validResults.length, errors: errors.length });
|
|
10073
|
-
return validResults;
|
|
10074
|
-
} catch (error) {
|
|
10075
|
-
if (error.message.includes("Partition '") && error.message.includes("' not found")) {
|
|
10076
|
-
console.warn(`Partition error in list method:`, error.message);
|
|
10077
|
-
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10078
|
-
return [];
|
|
10223
|
+
});
|
|
10224
|
+
return results.filter((item) => item !== null);
|
|
10225
|
+
}
|
|
10226
|
+
/**
|
|
10227
|
+
* Extract partition values from S3 key for specific ID
|
|
10228
|
+
*/
|
|
10229
|
+
extractPartitionValuesFromKey(id, keys, sortedFields) {
|
|
10230
|
+
const keyForId = keys.find((key) => key.includes(`id=${id}`));
|
|
10231
|
+
if (!keyForId) {
|
|
10232
|
+
throw new Error(`Partition key not found for ID ${id}`);
|
|
10233
|
+
}
|
|
10234
|
+
const keyParts = keyForId.split("/");
|
|
10235
|
+
const actualPartitionValues = {};
|
|
10236
|
+
for (const [fieldName] of sortedFields) {
|
|
10237
|
+
const fieldPart = keyParts.find((part) => part.startsWith(`${fieldName}=`));
|
|
10238
|
+
if (fieldPart) {
|
|
10239
|
+
const value = fieldPart.replace(`${fieldName}=`, "");
|
|
10240
|
+
actualPartitionValues[fieldName] = value;
|
|
10079
10241
|
}
|
|
10080
|
-
|
|
10242
|
+
}
|
|
10243
|
+
return actualPartitionValues;
|
|
10244
|
+
}
|
|
10245
|
+
/**
|
|
10246
|
+
* Handle resource-specific errors
|
|
10247
|
+
*/
|
|
10248
|
+
handleResourceError(error, id, context) {
|
|
10249
|
+
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
10250
|
+
console.warn(`Decryption failed for ${context} resource ${id}, returning basic info`);
|
|
10251
|
+
return {
|
|
10252
|
+
id,
|
|
10253
|
+
_decryptionFailed: true,
|
|
10254
|
+
_error: error.message,
|
|
10255
|
+
...context === "partition" && { _partition: context }
|
|
10256
|
+
};
|
|
10257
|
+
}
|
|
10258
|
+
throw error;
|
|
10259
|
+
}
|
|
10260
|
+
/**
|
|
10261
|
+
* Handle list method errors
|
|
10262
|
+
*/
|
|
10263
|
+
handleListError(error, { partition, partitionValues }) {
|
|
10264
|
+
if (error.message.includes("Partition '") && error.message.includes("' not found")) {
|
|
10265
|
+
console.warn(`Partition error in list method:`, error.message);
|
|
10081
10266
|
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10082
10267
|
return [];
|
|
10083
10268
|
}
|
|
10269
|
+
console.error(`Critical error in list method:`, error.message);
|
|
10270
|
+
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10271
|
+
return [];
|
|
10084
10272
|
}
|
|
10085
10273
|
/**
|
|
10086
10274
|
* Get multiple resources by their IDs
|
|
@@ -10428,30 +10616,14 @@ ${validation.errors.join("\n")}`);
|
|
|
10428
10616
|
for (const [partitionName, partition] of Object.entries(partitions)) {
|
|
10429
10617
|
const partitionKey = this.getPartitionKey({ partitionName, id: data.id, data });
|
|
10430
10618
|
if (partitionKey) {
|
|
10431
|
-
const mappedData = await this.schema.mapper(data);
|
|
10432
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10433
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleInsert({
|
|
10434
|
-
resource: this,
|
|
10435
|
-
data,
|
|
10436
|
-
mappedData
|
|
10437
|
-
});
|
|
10438
10619
|
const partitionMetadata = {
|
|
10439
|
-
|
|
10440
|
-
_version: this.version
|
|
10620
|
+
_v: String(this.version)
|
|
10441
10621
|
};
|
|
10442
|
-
let contentType = void 0;
|
|
10443
|
-
if (body && body !== "") {
|
|
10444
|
-
try {
|
|
10445
|
-
JSON.parse(body);
|
|
10446
|
-
contentType = "application/json";
|
|
10447
|
-
} catch {
|
|
10448
|
-
}
|
|
10449
|
-
}
|
|
10450
10622
|
await this.client.putObject({
|
|
10451
10623
|
key: partitionKey,
|
|
10452
10624
|
metadata: partitionMetadata,
|
|
10453
|
-
body,
|
|
10454
|
-
contentType
|
|
10625
|
+
body: "",
|
|
10626
|
+
contentType: void 0
|
|
10455
10627
|
});
|
|
10456
10628
|
}
|
|
10457
10629
|
}
|
|
@@ -10608,34 +10780,14 @@ ${validation.errors.join("\n")}`);
|
|
|
10608
10780
|
}
|
|
10609
10781
|
if (newPartitionKey) {
|
|
10610
10782
|
try {
|
|
10611
|
-
const mappedData = await this.schema.mapper(newData);
|
|
10612
|
-
if (mappedData.undefined !== void 0) delete mappedData.undefined;
|
|
10613
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10614
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
10615
|
-
resource: this,
|
|
10616
|
-
id,
|
|
10617
|
-
data: newData,
|
|
10618
|
-
mappedData
|
|
10619
|
-
});
|
|
10620
|
-
if (processedMetadata.undefined !== void 0) delete processedMetadata.undefined;
|
|
10621
10783
|
const partitionMetadata = {
|
|
10622
|
-
|
|
10623
|
-
_version: this.version
|
|
10784
|
+
_v: String(this.version)
|
|
10624
10785
|
};
|
|
10625
|
-
if (partitionMetadata.undefined !== void 0) delete partitionMetadata.undefined;
|
|
10626
|
-
let contentType = void 0;
|
|
10627
|
-
if (body && body !== "") {
|
|
10628
|
-
try {
|
|
10629
|
-
JSON.parse(body);
|
|
10630
|
-
contentType = "application/json";
|
|
10631
|
-
} catch {
|
|
10632
|
-
}
|
|
10633
|
-
}
|
|
10634
10786
|
await this.client.putObject({
|
|
10635
10787
|
key: newPartitionKey,
|
|
10636
10788
|
metadata: partitionMetadata,
|
|
10637
|
-
body,
|
|
10638
|
-
contentType
|
|
10789
|
+
body: "",
|
|
10790
|
+
contentType: void 0
|
|
10639
10791
|
});
|
|
10640
10792
|
} catch (error) {
|
|
10641
10793
|
console.warn(`New partition object could not be created for ${partitionName}:`, error.message);
|
|
@@ -10643,34 +10795,14 @@ ${validation.errors.join("\n")}`);
|
|
|
10643
10795
|
}
|
|
10644
10796
|
} else if (newPartitionKey) {
|
|
10645
10797
|
try {
|
|
10646
|
-
const mappedData = await this.schema.mapper(newData);
|
|
10647
|
-
if (mappedData.undefined !== void 0) delete mappedData.undefined;
|
|
10648
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10649
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
10650
|
-
resource: this,
|
|
10651
|
-
id,
|
|
10652
|
-
data: newData,
|
|
10653
|
-
mappedData
|
|
10654
|
-
});
|
|
10655
|
-
if (processedMetadata.undefined !== void 0) delete processedMetadata.undefined;
|
|
10656
10798
|
const partitionMetadata = {
|
|
10657
|
-
|
|
10658
|
-
_version: this.version
|
|
10799
|
+
_v: String(this.version)
|
|
10659
10800
|
};
|
|
10660
|
-
if (partitionMetadata.undefined !== void 0) delete partitionMetadata.undefined;
|
|
10661
|
-
let contentType = void 0;
|
|
10662
|
-
if (body && body !== "") {
|
|
10663
|
-
try {
|
|
10664
|
-
JSON.parse(body);
|
|
10665
|
-
contentType = "application/json";
|
|
10666
|
-
} catch {
|
|
10667
|
-
}
|
|
10668
|
-
}
|
|
10669
10801
|
await this.client.putObject({
|
|
10670
10802
|
key: newPartitionKey,
|
|
10671
10803
|
metadata: partitionMetadata,
|
|
10672
|
-
body,
|
|
10673
|
-
contentType
|
|
10804
|
+
body: "",
|
|
10805
|
+
contentType: void 0
|
|
10674
10806
|
});
|
|
10675
10807
|
} catch (error) {
|
|
10676
10808
|
console.warn(`Partition object could not be updated for ${partitionName}:`, error.message);
|
|
@@ -10693,32 +10825,15 @@ ${validation.errors.join("\n")}`);
|
|
|
10693
10825
|
}
|
|
10694
10826
|
const partitionKey = this.getPartitionKey({ partitionName, id: data.id, data });
|
|
10695
10827
|
if (partitionKey) {
|
|
10696
|
-
const mappedData = await this.schema.mapper(data);
|
|
10697
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10698
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
10699
|
-
resource: this,
|
|
10700
|
-
id: data.id,
|
|
10701
|
-
data,
|
|
10702
|
-
mappedData
|
|
10703
|
-
});
|
|
10704
10828
|
const partitionMetadata = {
|
|
10705
|
-
|
|
10706
|
-
_version: this.version
|
|
10829
|
+
_v: String(this.version)
|
|
10707
10830
|
};
|
|
10708
|
-
let contentType = void 0;
|
|
10709
|
-
if (body && body !== "") {
|
|
10710
|
-
try {
|
|
10711
|
-
JSON.parse(body);
|
|
10712
|
-
contentType = "application/json";
|
|
10713
|
-
} catch {
|
|
10714
|
-
}
|
|
10715
|
-
}
|
|
10716
10831
|
try {
|
|
10717
10832
|
await this.client.putObject({
|
|
10718
10833
|
key: partitionKey,
|
|
10719
10834
|
metadata: partitionMetadata,
|
|
10720
|
-
body,
|
|
10721
|
-
contentType
|
|
10835
|
+
body: "",
|
|
10836
|
+
contentType: void 0
|
|
10722
10837
|
});
|
|
10723
10838
|
} catch (error) {
|
|
10724
10839
|
console.warn(`Partition object could not be updated for ${partitionName}:`, error.message);
|
|
@@ -10768,40 +10883,75 @@ ${validation.errors.join("\n")}`);
|
|
|
10768
10883
|
throw new Error(`No partition values provided for partition '${partitionName}'`);
|
|
10769
10884
|
}
|
|
10770
10885
|
const partitionKey = join(`resource=${this.name}`, `partition=${partitionName}`, ...partitionSegments, `id=${id}`);
|
|
10771
|
-
|
|
10772
|
-
|
|
10773
|
-
|
|
10774
|
-
|
|
10775
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10776
|
-
let body = "";
|
|
10777
|
-
if (request.ContentLength > 0) {
|
|
10778
|
-
try {
|
|
10779
|
-
const fullObject = await this.client.getObject(partitionKey);
|
|
10780
|
-
body = await streamToString(fullObject.Body);
|
|
10781
|
-
} catch (error) {
|
|
10782
|
-
body = "";
|
|
10783
|
-
}
|
|
10886
|
+
try {
|
|
10887
|
+
await this.client.headObject(partitionKey);
|
|
10888
|
+
} catch (error) {
|
|
10889
|
+
throw new Error(`Resource with id '${id}' not found in partition '${partitionName}'`);
|
|
10784
10890
|
}
|
|
10785
|
-
const
|
|
10786
|
-
resource: this,
|
|
10787
|
-
metadata,
|
|
10788
|
-
body
|
|
10789
|
-
});
|
|
10790
|
-
let data = processedMetadata;
|
|
10791
|
-
data.id = id;
|
|
10792
|
-
data._contentLength = request.ContentLength;
|
|
10793
|
-
data._lastModified = request.LastModified;
|
|
10794
|
-
data._hasContent = request.ContentLength > 0;
|
|
10795
|
-
data._mimeType = request.ContentType || null;
|
|
10891
|
+
const data = await this.get(id);
|
|
10796
10892
|
data._partition = partitionName;
|
|
10797
10893
|
data._partitionValues = partitionValues;
|
|
10798
|
-
if (request.VersionId) data._versionId = request.VersionId;
|
|
10799
|
-
if (request.Expiration) data._expiresAt = request.Expiration;
|
|
10800
|
-
data._definitionHash = this.getDefinitionHash();
|
|
10801
|
-
if (data.undefined !== void 0) delete data.undefined;
|
|
10802
10894
|
this.emit("getFromPartition", data);
|
|
10803
10895
|
return data;
|
|
10804
10896
|
}
|
|
10897
|
+
/**
|
|
10898
|
+
* Create a historical version of an object
|
|
10899
|
+
* @param {string} id - Resource ID
|
|
10900
|
+
* @param {Object} data - Object data to store historically
|
|
10901
|
+
*/
|
|
10902
|
+
async createHistoricalVersion(id, data) {
|
|
10903
|
+
const historicalKey = join(`resource=${this.name}`, `historical`, `id=${id}`);
|
|
10904
|
+
const historicalData = {
|
|
10905
|
+
...data,
|
|
10906
|
+
_v: data._v || this.version,
|
|
10907
|
+
_historicalTimestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
10908
|
+
};
|
|
10909
|
+
const mappedData = await this.schema.mapper(historicalData);
|
|
10910
|
+
const behaviorImpl = getBehavior(this.behavior);
|
|
10911
|
+
const { mappedData: processedMetadata, body } = await behaviorImpl.handleInsert({
|
|
10912
|
+
resource: this,
|
|
10913
|
+
data: historicalData,
|
|
10914
|
+
mappedData
|
|
10915
|
+
});
|
|
10916
|
+
const finalMetadata = {
|
|
10917
|
+
...processedMetadata,
|
|
10918
|
+
_v: data._v || this.version,
|
|
10919
|
+
_historicalTimestamp: historicalData._historicalTimestamp
|
|
10920
|
+
};
|
|
10921
|
+
let contentType = void 0;
|
|
10922
|
+
if (body && body !== "") {
|
|
10923
|
+
try {
|
|
10924
|
+
JSON.parse(body);
|
|
10925
|
+
contentType = "application/json";
|
|
10926
|
+
} catch {
|
|
10927
|
+
}
|
|
10928
|
+
}
|
|
10929
|
+
await this.client.putObject({
|
|
10930
|
+
key: historicalKey,
|
|
10931
|
+
metadata: finalMetadata,
|
|
10932
|
+
body,
|
|
10933
|
+
contentType
|
|
10934
|
+
});
|
|
10935
|
+
}
|
|
10936
|
+
/**
|
|
10937
|
+
* Apply version mapping to convert an object from one version to another
|
|
10938
|
+
* @param {Object} data - Object data to map
|
|
10939
|
+
* @param {string} fromVersion - Source version
|
|
10940
|
+
* @param {string} toVersion - Target version
|
|
10941
|
+
* @returns {Object} Mapped object data
|
|
10942
|
+
*/
|
|
10943
|
+
async applyVersionMapping(data, fromVersion, toVersion) {
|
|
10944
|
+
if (fromVersion === toVersion) {
|
|
10945
|
+
return data;
|
|
10946
|
+
}
|
|
10947
|
+
const mappedData = {
|
|
10948
|
+
...data,
|
|
10949
|
+
_v: toVersion,
|
|
10950
|
+
_originalVersion: fromVersion,
|
|
10951
|
+
_versionMapped: true
|
|
10952
|
+
};
|
|
10953
|
+
return mappedData;
|
|
10954
|
+
}
|
|
10805
10955
|
}
|
|
10806
10956
|
function validateResourceConfig(config) {
|
|
10807
10957
|
const errors = [];
|
|
@@ -10916,7 +11066,7 @@ class Database extends EventEmitter {
|
|
|
10916
11066
|
this.version = "1";
|
|
10917
11067
|
this.s3dbVersion = (() => {
|
|
10918
11068
|
try {
|
|
10919
|
-
return true ? "
|
|
11069
|
+
return true ? "6.1.0" : "latest";
|
|
10920
11070
|
} catch (e) {
|
|
10921
11071
|
return "latest";
|
|
10922
11072
|
}
|
|
@@ -10927,8 +11077,10 @@ class Database extends EventEmitter {
|
|
|
10927
11077
|
this.verbose = options.verbose || false;
|
|
10928
11078
|
this.parallelism = parseInt(options.parallelism + "") || 10;
|
|
10929
11079
|
this.plugins = options.plugins || [];
|
|
11080
|
+
this.pluginList = options.plugins || [];
|
|
10930
11081
|
this.cache = options.cache;
|
|
10931
11082
|
this.passphrase = options.passphrase || "secret";
|
|
11083
|
+
this.versioningEnabled = options.versioningEnabled || false;
|
|
10932
11084
|
let connectionString = options.connectionString;
|
|
10933
11085
|
if (!connectionString && (options.bucket || options.accessKeyId || options.secretAccessKey)) {
|
|
10934
11086
|
const { bucket, region, accessKeyId, secretAccessKey, endpoint, forcePathStyle } = options;
|
|
@@ -10979,7 +11131,7 @@ class Database extends EventEmitter {
|
|
|
10979
11131
|
client: this.client,
|
|
10980
11132
|
version: currentVersion,
|
|
10981
11133
|
attributes: versionData.attributes,
|
|
10982
|
-
behavior: versionData.behavior || "user-
|
|
11134
|
+
behavior: versionData.behavior || "user-managed",
|
|
10983
11135
|
parallelism: this.parallelism,
|
|
10984
11136
|
passphrase: this.passphrase,
|
|
10985
11137
|
observers: [this],
|
|
@@ -10989,7 +11141,8 @@ class Database extends EventEmitter {
|
|
|
10989
11141
|
paranoid: versionData.paranoid !== void 0 ? versionData.paranoid : true,
|
|
10990
11142
|
allNestedObjectsOptional: versionData.allNestedObjectsOptional !== void 0 ? versionData.allNestedObjectsOptional : true,
|
|
10991
11143
|
autoDecrypt: versionData.autoDecrypt !== void 0 ? versionData.autoDecrypt : true,
|
|
10992
|
-
hooks: versionData.hooks || {}
|
|
11144
|
+
hooks: versionData.hooks || {},
|
|
11145
|
+
versioningEnabled: this.versioningEnabled
|
|
10993
11146
|
});
|
|
10994
11147
|
}
|
|
10995
11148
|
}
|
|
@@ -11064,7 +11217,7 @@ class Database extends EventEmitter {
|
|
|
11064
11217
|
}
|
|
11065
11218
|
const hashObj = {
|
|
11066
11219
|
attributes: stableAttributes,
|
|
11067
|
-
behavior: behavior || definition.behavior || "user-
|
|
11220
|
+
behavior: behavior || definition.behavior || "user-managed"
|
|
11068
11221
|
};
|
|
11069
11222
|
const stableString = jsonStableStringify(hashObj);
|
|
11070
11223
|
return `sha256:${crypto.createHash("sha256").update(stableString).digest("hex")}`;
|
|
@@ -11081,8 +11234,8 @@ class Database extends EventEmitter {
|
|
|
11081
11234
|
}
|
|
11082
11235
|
async startPlugins() {
|
|
11083
11236
|
const db = this;
|
|
11084
|
-
if (!lodashEs.isEmpty(this.
|
|
11085
|
-
const plugins = this.
|
|
11237
|
+
if (!lodashEs.isEmpty(this.pluginList)) {
|
|
11238
|
+
const plugins = this.pluginList.map((p) => lodashEs.isFunction(p) ? new p(this) : p);
|
|
11086
11239
|
const setupProms = plugins.map(async (plugin) => {
|
|
11087
11240
|
if (plugin.beforeSetup) await plugin.beforeSetup();
|
|
11088
11241
|
await plugin.setup(db);
|
|
@@ -11097,6 +11250,20 @@ class Database extends EventEmitter {
|
|
|
11097
11250
|
await Promise.all(startProms);
|
|
11098
11251
|
}
|
|
11099
11252
|
}
|
|
11253
|
+
/**
|
|
11254
|
+
* Register and setup a plugin
|
|
11255
|
+
* @param {Plugin} plugin - Plugin instance to register
|
|
11256
|
+
* @param {string} [name] - Optional name for the plugin (defaults to plugin.constructor.name)
|
|
11257
|
+
*/
|
|
11258
|
+
async usePlugin(plugin, name = null) {
|
|
11259
|
+
const pluginName = name || plugin.constructor.name.replace("Plugin", "").toLowerCase();
|
|
11260
|
+
this.plugins[pluginName] = plugin;
|
|
11261
|
+
if (this.isConnected()) {
|
|
11262
|
+
await plugin.setup(this);
|
|
11263
|
+
await plugin.start();
|
|
11264
|
+
}
|
|
11265
|
+
return plugin;
|
|
11266
|
+
}
|
|
11100
11267
|
async uploadMetadataFile() {
|
|
11101
11268
|
const metadata = {
|
|
11102
11269
|
version: this.version,
|
|
@@ -11127,7 +11294,7 @@ class Database extends EventEmitter {
|
|
|
11127
11294
|
[version]: {
|
|
11128
11295
|
hash: definitionHash,
|
|
11129
11296
|
attributes: resourceDef.attributes,
|
|
11130
|
-
behavior: resourceDef.behavior || "user-
|
|
11297
|
+
behavior: resourceDef.behavior || "user-managed",
|
|
11131
11298
|
timestamps: resource.config.timestamps,
|
|
11132
11299
|
partitions: resource.config.partitions,
|
|
11133
11300
|
paranoid: resource.config.paranoid,
|
|
@@ -11176,7 +11343,7 @@ class Database extends EventEmitter {
|
|
|
11176
11343
|
* @param {Object} [config.options] - Resource options (deprecated, use root level parameters)
|
|
11177
11344
|
* @returns {Object} Result with exists and hash information
|
|
11178
11345
|
*/
|
|
11179
|
-
resourceExistsWithSameHash({ name, attributes, behavior = "user-
|
|
11346
|
+
resourceExistsWithSameHash({ name, attributes, behavior = "user-managed", options = {} }) {
|
|
11180
11347
|
if (!this.resources[name]) {
|
|
11181
11348
|
return { exists: false, sameHash: false, hash: null };
|
|
11182
11349
|
}
|
|
@@ -11189,6 +11356,7 @@ class Database extends EventEmitter {
|
|
|
11189
11356
|
client: this.client,
|
|
11190
11357
|
version: existingResource.version,
|
|
11191
11358
|
passphrase: this.passphrase,
|
|
11359
|
+
versioningEnabled: this.versioningEnabled,
|
|
11192
11360
|
...options
|
|
11193
11361
|
});
|
|
11194
11362
|
const newHash = this.generateDefinitionHash(mockResource.export());
|
|
@@ -11199,7 +11367,7 @@ class Database extends EventEmitter {
|
|
|
11199
11367
|
existingHash
|
|
11200
11368
|
};
|
|
11201
11369
|
}
|
|
11202
|
-
async createResource({ name, attributes, behavior = "user-
|
|
11370
|
+
async createResource({ name, attributes, behavior = "user-managed", hooks, ...config }) {
|
|
11203
11371
|
if (this.resources[name]) {
|
|
11204
11372
|
const existingResource = this.resources[name];
|
|
11205
11373
|
Object.assign(existingResource.config, {
|
|
@@ -11209,6 +11377,7 @@ class Database extends EventEmitter {
|
|
|
11209
11377
|
if (behavior) {
|
|
11210
11378
|
existingResource.behavior = behavior;
|
|
11211
11379
|
}
|
|
11380
|
+
existingResource.versioningEnabled = this.versioningEnabled;
|
|
11212
11381
|
existingResource.updateAttributes(attributes);
|
|
11213
11382
|
if (hooks) {
|
|
11214
11383
|
for (const [event, hooksArr] of Object.entries(hooks)) {
|
|
@@ -11243,6 +11412,7 @@ class Database extends EventEmitter {
|
|
|
11243
11412
|
passphrase: this.passphrase,
|
|
11244
11413
|
cache: this.cache,
|
|
11245
11414
|
hooks,
|
|
11415
|
+
versioningEnabled: this.versioningEnabled,
|
|
11246
11416
|
...config
|
|
11247
11417
|
});
|
|
11248
11418
|
this.resources[name] = resource;
|
|
@@ -17709,12 +17879,117 @@ class S3Cache extends Cache {
|
|
|
17709
17879
|
}
|
|
17710
17880
|
|
|
17711
17881
|
class Plugin extends EventEmitter {
|
|
17882
|
+
constructor(options = {}) {
|
|
17883
|
+
super();
|
|
17884
|
+
this.name = this.constructor.name;
|
|
17885
|
+
this.options = options;
|
|
17886
|
+
this.hooks = /* @__PURE__ */ new Map();
|
|
17887
|
+
}
|
|
17712
17888
|
async setup(database) {
|
|
17889
|
+
this.database = database;
|
|
17890
|
+
this.beforeSetup();
|
|
17891
|
+
await this.onSetup();
|
|
17892
|
+
this.afterSetup();
|
|
17713
17893
|
}
|
|
17714
17894
|
async start() {
|
|
17895
|
+
this.beforeStart();
|
|
17896
|
+
await this.onStart();
|
|
17897
|
+
this.afterStart();
|
|
17715
17898
|
}
|
|
17716
17899
|
async stop() {
|
|
17900
|
+
this.beforeStop();
|
|
17901
|
+
await this.onStop();
|
|
17902
|
+
this.afterStop();
|
|
17903
|
+
}
|
|
17904
|
+
// Override these methods in subclasses
|
|
17905
|
+
async onSetup() {
|
|
17906
|
+
}
|
|
17907
|
+
async onStart() {
|
|
17717
17908
|
}
|
|
17909
|
+
async onStop() {
|
|
17910
|
+
}
|
|
17911
|
+
// Hook management methods
|
|
17912
|
+
addHook(resource, event, handler) {
|
|
17913
|
+
if (!this.hooks.has(resource)) {
|
|
17914
|
+
this.hooks.set(resource, /* @__PURE__ */ new Map());
|
|
17915
|
+
}
|
|
17916
|
+
const resourceHooks = this.hooks.get(resource);
|
|
17917
|
+
if (!resourceHooks.has(event)) {
|
|
17918
|
+
resourceHooks.set(event, []);
|
|
17919
|
+
}
|
|
17920
|
+
resourceHooks.get(event).push(handler);
|
|
17921
|
+
}
|
|
17922
|
+
removeHook(resource, event, handler) {
|
|
17923
|
+
const resourceHooks = this.hooks.get(resource);
|
|
17924
|
+
if (resourceHooks && resourceHooks.has(event)) {
|
|
17925
|
+
const handlers = resourceHooks.get(event);
|
|
17926
|
+
const index = handlers.indexOf(handler);
|
|
17927
|
+
if (index > -1) {
|
|
17928
|
+
handlers.splice(index, 1);
|
|
17929
|
+
}
|
|
17930
|
+
}
|
|
17931
|
+
}
|
|
17932
|
+
// Enhanced resource method wrapping that supports multiple plugins
|
|
17933
|
+
wrapResourceMethod(resource, methodName, wrapper) {
|
|
17934
|
+
const originalMethod = resource[methodName];
|
|
17935
|
+
if (!resource._pluginWrappers) {
|
|
17936
|
+
resource._pluginWrappers = /* @__PURE__ */ new Map();
|
|
17937
|
+
}
|
|
17938
|
+
if (!resource._pluginWrappers.has(methodName)) {
|
|
17939
|
+
resource._pluginWrappers.set(methodName, []);
|
|
17940
|
+
}
|
|
17941
|
+
resource._pluginWrappers.get(methodName).push(wrapper);
|
|
17942
|
+
if (!resource[`_wrapped_${methodName}`]) {
|
|
17943
|
+
resource[`_wrapped_${methodName}`] = originalMethod;
|
|
17944
|
+
const isJestMock = originalMethod && originalMethod._isMockFunction;
|
|
17945
|
+
resource[methodName] = async function(...args) {
|
|
17946
|
+
let result = await resource[`_wrapped_${methodName}`](...args);
|
|
17947
|
+
for (const wrapper2 of resource._pluginWrappers.get(methodName)) {
|
|
17948
|
+
result = await wrapper2.call(this, result, args, methodName);
|
|
17949
|
+
}
|
|
17950
|
+
return result;
|
|
17951
|
+
};
|
|
17952
|
+
if (isJestMock) {
|
|
17953
|
+
Object.setPrototypeOf(resource[methodName], Object.getPrototypeOf(originalMethod));
|
|
17954
|
+
Object.assign(resource[methodName], originalMethod);
|
|
17955
|
+
}
|
|
17956
|
+
}
|
|
17957
|
+
}
|
|
17958
|
+
// Partition-aware helper methods
|
|
17959
|
+
getPartitionValues(data, resource) {
|
|
17960
|
+
if (!resource.config?.partitions) return {};
|
|
17961
|
+
const partitionValues = {};
|
|
17962
|
+
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
17963
|
+
if (partitionDef.fields) {
|
|
17964
|
+
partitionValues[partitionName] = {};
|
|
17965
|
+
for (const [fieldName, rule] of Object.entries(partitionDef.fields)) {
|
|
17966
|
+
const value = this.getNestedFieldValue(data, fieldName);
|
|
17967
|
+
if (value !== null && value !== void 0) {
|
|
17968
|
+
partitionValues[partitionName][fieldName] = resource.applyPartitionRule(value, rule);
|
|
17969
|
+
}
|
|
17970
|
+
}
|
|
17971
|
+
} else {
|
|
17972
|
+
partitionValues[partitionName] = {};
|
|
17973
|
+
}
|
|
17974
|
+
}
|
|
17975
|
+
return partitionValues;
|
|
17976
|
+
}
|
|
17977
|
+
getNestedFieldValue(data, fieldPath) {
|
|
17978
|
+
if (!fieldPath.includes(".")) {
|
|
17979
|
+
return data[fieldPath] ?? null;
|
|
17980
|
+
}
|
|
17981
|
+
const keys = fieldPath.split(".");
|
|
17982
|
+
let value = data;
|
|
17983
|
+
for (const key of keys) {
|
|
17984
|
+
if (value && typeof value === "object" && key in value) {
|
|
17985
|
+
value = value[key];
|
|
17986
|
+
} else {
|
|
17987
|
+
return null;
|
|
17988
|
+
}
|
|
17989
|
+
}
|
|
17990
|
+
return value ?? null;
|
|
17991
|
+
}
|
|
17992
|
+
// Event emission methods
|
|
17718
17993
|
beforeSetup() {
|
|
17719
17994
|
this.emit("plugin.beforeSetup", /* @__PURE__ */ new Date());
|
|
17720
17995
|
}
|
|
@@ -17744,208 +18019,3066 @@ const PluginObject = {
|
|
|
17744
18019
|
}
|
|
17745
18020
|
};
|
|
17746
18021
|
|
|
17747
|
-
|
|
17748
|
-
async setup(db) {
|
|
17749
|
-
this.client = db.client;
|
|
17750
|
-
this.map = {
|
|
17751
|
-
PutObjectCommand: "put",
|
|
17752
|
-
GetObjectCommand: "get",
|
|
17753
|
-
HeadObjectCommand: "get",
|
|
17754
|
-
DeleteObjectCommand: "delete",
|
|
17755
|
-
DeleteObjectsCommand: "delete",
|
|
17756
|
-
ListObjectsV2Command: "list"
|
|
17757
|
-
};
|
|
17758
|
-
this.costs = {
|
|
17759
|
-
total: 0,
|
|
17760
|
-
prices: {
|
|
17761
|
-
put: 5e-3 / 1e3,
|
|
17762
|
-
copy: 5e-3 / 1e3,
|
|
17763
|
-
list: 5e-3 / 1e3,
|
|
17764
|
-
post: 5e-3 / 1e3,
|
|
17765
|
-
get: 4e-4 / 1e3,
|
|
17766
|
-
select: 4e-4 / 1e3,
|
|
17767
|
-
delete: 4e-4 / 1e3
|
|
17768
|
-
},
|
|
17769
|
-
requests: {
|
|
17770
|
-
total: 0,
|
|
17771
|
-
put: 0,
|
|
17772
|
-
post: 0,
|
|
17773
|
-
copy: 0,
|
|
17774
|
-
list: 0,
|
|
17775
|
-
get: 0,
|
|
17776
|
-
select: 0,
|
|
17777
|
-
delete: 0
|
|
17778
|
-
},
|
|
17779
|
-
events: {
|
|
17780
|
-
total: 0,
|
|
17781
|
-
PutObjectCommand: 0,
|
|
17782
|
-
GetObjectCommand: 0,
|
|
17783
|
-
HeadObjectCommand: 0,
|
|
17784
|
-
DeleteObjectCommand: 0,
|
|
17785
|
-
DeleteObjectsCommand: 0,
|
|
17786
|
-
ListObjectsV2Command: 0
|
|
17787
|
-
}
|
|
17788
|
-
};
|
|
17789
|
-
this.client.costs = JSON.parse(JSON.stringify(this.costs));
|
|
17790
|
-
},
|
|
17791
|
-
async start() {
|
|
17792
|
-
this.client.on("command.response", (name) => this.addRequest(name, this.map[name]));
|
|
17793
|
-
},
|
|
17794
|
-
addRequest(name, method) {
|
|
17795
|
-
this.costs.events[name]++;
|
|
17796
|
-
this.costs.events.total++;
|
|
17797
|
-
this.costs.requests.total++;
|
|
17798
|
-
this.costs.requests[method]++;
|
|
17799
|
-
this.costs.total += this.costs.prices[method];
|
|
17800
|
-
this.client.costs.events[name]++;
|
|
17801
|
-
this.client.costs.events.total++;
|
|
17802
|
-
this.client.costs.requests.total++;
|
|
17803
|
-
this.client.costs.requests[method]++;
|
|
17804
|
-
this.client.costs.total += this.client.costs.prices[method];
|
|
17805
|
-
}
|
|
17806
|
-
};
|
|
17807
|
-
|
|
17808
|
-
class CachePlugin extends Plugin {
|
|
18022
|
+
class AuditPlugin extends Plugin {
|
|
17809
18023
|
constructor(options = {}) {
|
|
17810
|
-
super();
|
|
17811
|
-
this.
|
|
18024
|
+
super(options);
|
|
18025
|
+
this.auditResource = null;
|
|
18026
|
+
this.config = {
|
|
18027
|
+
enabled: options.enabled !== false,
|
|
18028
|
+
includeData: options.includeData !== false,
|
|
18029
|
+
includePartitions: options.includePartitions !== false,
|
|
18030
|
+
maxDataSize: options.maxDataSize || 1e4,
|
|
18031
|
+
// 10KB limit
|
|
18032
|
+
...options
|
|
18033
|
+
};
|
|
17812
18034
|
}
|
|
17813
|
-
async
|
|
17814
|
-
this.
|
|
17815
|
-
|
|
17816
|
-
|
|
17817
|
-
|
|
17818
|
-
|
|
17819
|
-
|
|
17820
|
-
|
|
17821
|
-
|
|
18035
|
+
async onSetup() {
|
|
18036
|
+
if (!this.config.enabled) {
|
|
18037
|
+
this.auditResource = null;
|
|
18038
|
+
return;
|
|
18039
|
+
}
|
|
18040
|
+
try {
|
|
18041
|
+
this.auditResource = await this.database.createResource({
|
|
18042
|
+
name: "audits",
|
|
18043
|
+
attributes: {
|
|
18044
|
+
id: "string|required",
|
|
18045
|
+
resourceName: "string|required",
|
|
18046
|
+
operation: "string|required",
|
|
18047
|
+
recordId: "string|required",
|
|
18048
|
+
userId: "string|optional",
|
|
18049
|
+
timestamp: "string|required",
|
|
18050
|
+
oldData: "string|optional",
|
|
18051
|
+
newData: "string|optional",
|
|
18052
|
+
partition: "string|optional",
|
|
18053
|
+
partitionValues: "string|optional",
|
|
18054
|
+
metadata: "string|optional"
|
|
18055
|
+
}
|
|
18056
|
+
});
|
|
18057
|
+
} catch (error) {
|
|
18058
|
+
try {
|
|
18059
|
+
this.auditResource = this.database.resources.audits;
|
|
18060
|
+
} catch (innerError) {
|
|
18061
|
+
this.auditResource = null;
|
|
18062
|
+
return;
|
|
18063
|
+
}
|
|
17822
18064
|
}
|
|
18065
|
+
this.installDatabaseProxy();
|
|
18066
|
+
this.installResourceHooks();
|
|
17823
18067
|
}
|
|
17824
|
-
async
|
|
18068
|
+
async onStart() {
|
|
17825
18069
|
}
|
|
17826
|
-
async
|
|
18070
|
+
async onStop() {
|
|
17827
18071
|
}
|
|
17828
18072
|
installDatabaseProxy() {
|
|
17829
|
-
|
|
17830
|
-
|
|
18073
|
+
if (this.database._auditProxyInstalled) {
|
|
18074
|
+
return;
|
|
18075
|
+
}
|
|
18076
|
+
const installResourceHooksForResource = this.installResourceHooksForResource.bind(this);
|
|
18077
|
+
this.database._originalCreateResource = this.database.createResource;
|
|
17831
18078
|
this.database.createResource = async function(...args) {
|
|
17832
|
-
const resource = await this.
|
|
17833
|
-
|
|
18079
|
+
const resource = await this._originalCreateResource(...args);
|
|
18080
|
+
if (resource.name !== "audit_logs") {
|
|
18081
|
+
installResourceHooksForResource(resource);
|
|
18082
|
+
}
|
|
17834
18083
|
return resource;
|
|
17835
18084
|
};
|
|
17836
|
-
|
|
17837
|
-
|
|
17838
|
-
|
|
17839
|
-
|
|
17840
|
-
|
|
17841
|
-
|
|
17842
|
-
|
|
17843
|
-
|
|
17844
|
-
|
|
17845
|
-
|
|
17846
|
-
|
|
17847
|
-
|
|
17848
|
-
|
|
17849
|
-
|
|
17850
|
-
|
|
17851
|
-
|
|
17852
|
-
|
|
17853
|
-
|
|
17854
|
-
|
|
17855
|
-
|
|
17856
|
-
|
|
17857
|
-
|
|
17858
|
-
|
|
17859
|
-
|
|
17860
|
-
|
|
17861
|
-
|
|
18085
|
+
this.database._auditProxyInstalled = true;
|
|
18086
|
+
}
|
|
18087
|
+
installResourceHooks() {
|
|
18088
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18089
|
+
if (resource.name === "audits") continue;
|
|
18090
|
+
this.installResourceHooksForResource(resource);
|
|
18091
|
+
}
|
|
18092
|
+
}
|
|
18093
|
+
installResourceHooksForResource(resource) {
|
|
18094
|
+
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
18095
|
+
const [data] = args;
|
|
18096
|
+
const recordId = data.id || result.id || "auto-generated";
|
|
18097
|
+
const partitionValues = this.config.includePartitions ? this.getPartitionValues(data, resource) : null;
|
|
18098
|
+
const auditRecord = {
|
|
18099
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18100
|
+
resourceName: resource.name,
|
|
18101
|
+
operation: "insert",
|
|
18102
|
+
recordId,
|
|
18103
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18104
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18105
|
+
oldData: null,
|
|
18106
|
+
newData: this.config.includeData === false ? null : JSON.stringify(this.truncateData(data)),
|
|
18107
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18108
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18109
|
+
metadata: JSON.stringify({
|
|
18110
|
+
source: "audit-plugin",
|
|
18111
|
+
version: "2.0"
|
|
18112
|
+
})
|
|
18113
|
+
};
|
|
18114
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18115
|
+
return result;
|
|
18116
|
+
});
|
|
18117
|
+
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
18118
|
+
const [id, data] = args;
|
|
18119
|
+
let oldData = null;
|
|
18120
|
+
if (this.config.includeData) {
|
|
18121
|
+
try {
|
|
18122
|
+
oldData = await resource.get(id);
|
|
18123
|
+
} catch (error) {
|
|
18124
|
+
}
|
|
17862
18125
|
}
|
|
17863
|
-
const
|
|
17864
|
-
|
|
17865
|
-
|
|
17866
|
-
|
|
17867
|
-
|
|
17868
|
-
|
|
17869
|
-
|
|
17870
|
-
|
|
17871
|
-
|
|
17872
|
-
|
|
17873
|
-
|
|
18126
|
+
const partitionValues = this.config.includePartitions ? this.getPartitionValues(result, resource) : null;
|
|
18127
|
+
const auditRecord = {
|
|
18128
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18129
|
+
resourceName: resource.name,
|
|
18130
|
+
operation: "update",
|
|
18131
|
+
recordId: id,
|
|
18132
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18133
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18134
|
+
oldData: oldData && this.config.includeData === false ? null : oldData ? JSON.stringify(this.truncateData(oldData)) : null,
|
|
18135
|
+
newData: this.config.includeData === false ? null : JSON.stringify(this.truncateData(result)),
|
|
18136
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18137
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18138
|
+
metadata: JSON.stringify({
|
|
18139
|
+
source: "audit-plugin",
|
|
18140
|
+
version: "2.0"
|
|
18141
|
+
})
|
|
18142
|
+
};
|
|
18143
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18144
|
+
return result;
|
|
18145
|
+
});
|
|
18146
|
+
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
18147
|
+
const [id] = args;
|
|
18148
|
+
let oldData = null;
|
|
18149
|
+
if (this.config.includeData) {
|
|
18150
|
+
try {
|
|
18151
|
+
oldData = await resource.get(id);
|
|
18152
|
+
} catch (error) {
|
|
18153
|
+
}
|
|
17874
18154
|
}
|
|
17875
|
-
const
|
|
17876
|
-
|
|
17877
|
-
|
|
17878
|
-
|
|
17879
|
-
|
|
17880
|
-
|
|
17881
|
-
|
|
17882
|
-
|
|
17883
|
-
|
|
17884
|
-
|
|
17885
|
-
|
|
17886
|
-
|
|
17887
|
-
|
|
17888
|
-
|
|
18155
|
+
const partitionValues = oldData && this.config.includePartitions ? this.getPartitionValues(oldData, resource) : null;
|
|
18156
|
+
const auditRecord = {
|
|
18157
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18158
|
+
resourceName: resource.name,
|
|
18159
|
+
operation: "delete",
|
|
18160
|
+
recordId: id,
|
|
18161
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18162
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18163
|
+
oldData: oldData && this.config.includeData === false ? null : oldData ? JSON.stringify(this.truncateData(oldData)) : null,
|
|
18164
|
+
newData: null,
|
|
18165
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18166
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18167
|
+
metadata: JSON.stringify({
|
|
18168
|
+
source: "audit-plugin",
|
|
18169
|
+
version: "2.0"
|
|
18170
|
+
})
|
|
18171
|
+
};
|
|
18172
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18173
|
+
return result;
|
|
18174
|
+
});
|
|
18175
|
+
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
18176
|
+
const [ids] = args;
|
|
18177
|
+
const auditRecords = [];
|
|
18178
|
+
if (this.config.includeData) {
|
|
18179
|
+
for (const id of ids) {
|
|
18180
|
+
try {
|
|
18181
|
+
const oldData = await resource.get(id);
|
|
18182
|
+
const partitionValues = this.config.includePartitions ? this.getPartitionValues(oldData, resource) : null;
|
|
18183
|
+
auditRecords.push({
|
|
18184
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18185
|
+
resourceName: resource.name,
|
|
18186
|
+
operation: "delete",
|
|
18187
|
+
recordId: id,
|
|
18188
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18189
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18190
|
+
oldData: this.config.includeData === false ? null : JSON.stringify(this.truncateData(oldData)),
|
|
18191
|
+
newData: null,
|
|
18192
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18193
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18194
|
+
metadata: JSON.stringify({
|
|
18195
|
+
source: "audit-plugin",
|
|
18196
|
+
version: "2.0",
|
|
18197
|
+
batchOperation: true
|
|
18198
|
+
})
|
|
18199
|
+
});
|
|
18200
|
+
} catch (error) {
|
|
18201
|
+
}
|
|
18202
|
+
}
|
|
17889
18203
|
}
|
|
17890
|
-
const
|
|
17891
|
-
|
|
17892
|
-
|
|
17893
|
-
|
|
17894
|
-
|
|
17895
|
-
|
|
17896
|
-
|
|
17897
|
-
|
|
17898
|
-
|
|
17899
|
-
|
|
17900
|
-
|
|
18204
|
+
for (const auditRecord of auditRecords) {
|
|
18205
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18206
|
+
}
|
|
18207
|
+
return result;
|
|
18208
|
+
});
|
|
18209
|
+
}
|
|
18210
|
+
getPartitionValues(data, resource) {
|
|
18211
|
+
const partitions = resource.config?.partitions || {};
|
|
18212
|
+
const partitionValues = {};
|
|
18213
|
+
for (const [partitionName, partitionDef] of Object.entries(partitions)) {
|
|
18214
|
+
if (partitionDef.fields) {
|
|
18215
|
+
const partitionData = {};
|
|
18216
|
+
for (const [fieldName, fieldRule] of Object.entries(partitionDef.fields)) {
|
|
18217
|
+
const fieldValue = this.getNestedFieldValue(data, fieldName);
|
|
18218
|
+
if (fieldValue !== void 0 && fieldValue !== null) {
|
|
18219
|
+
partitionData[fieldName] = fieldValue;
|
|
18220
|
+
}
|
|
18221
|
+
}
|
|
18222
|
+
if (Object.keys(partitionData).length > 0) {
|
|
18223
|
+
partitionValues[partitionName] = partitionData;
|
|
18224
|
+
}
|
|
18225
|
+
}
|
|
18226
|
+
}
|
|
18227
|
+
return partitionValues;
|
|
18228
|
+
}
|
|
18229
|
+
getNestedFieldValue(data, fieldPath) {
|
|
18230
|
+
if (!fieldPath.includes(".")) {
|
|
18231
|
+
return data[fieldPath];
|
|
18232
|
+
}
|
|
18233
|
+
const keys = fieldPath.split(".");
|
|
18234
|
+
let currentLevel = data;
|
|
18235
|
+
for (const key of keys) {
|
|
18236
|
+
if (!currentLevel || typeof currentLevel !== "object" || !(key in currentLevel)) {
|
|
18237
|
+
return void 0;
|
|
17901
18238
|
}
|
|
17902
|
-
|
|
17903
|
-
|
|
18239
|
+
currentLevel = currentLevel[key];
|
|
18240
|
+
}
|
|
18241
|
+
return currentLevel;
|
|
18242
|
+
}
|
|
18243
|
+
getPrimaryPartition(partitionValues) {
|
|
18244
|
+
if (!partitionValues) return null;
|
|
18245
|
+
const partitionNames = Object.keys(partitionValues);
|
|
18246
|
+
return partitionNames.length > 0 ? partitionNames[0] : null;
|
|
18247
|
+
}
|
|
18248
|
+
async logAudit(auditRecord) {
|
|
18249
|
+
if (!this.auditResource) return;
|
|
18250
|
+
try {
|
|
18251
|
+
await this.auditResource.insert(auditRecord);
|
|
18252
|
+
} catch (error) {
|
|
18253
|
+
console.error("Failed to log audit record:", error);
|
|
18254
|
+
if (error && error.stack) console.error(error.stack);
|
|
18255
|
+
}
|
|
18256
|
+
}
|
|
18257
|
+
truncateData(data) {
|
|
18258
|
+
if (!data) return data;
|
|
18259
|
+
const dataStr = JSON.stringify(data);
|
|
18260
|
+
if (dataStr.length <= this.config.maxDataSize) {
|
|
17904
18261
|
return data;
|
|
18262
|
+
}
|
|
18263
|
+
return {
|
|
18264
|
+
...data,
|
|
18265
|
+
_truncated: true,
|
|
18266
|
+
_originalSize: dataStr.length,
|
|
18267
|
+
_truncatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17905
18268
|
};
|
|
17906
|
-
|
|
17907
|
-
|
|
17908
|
-
|
|
17909
|
-
|
|
17910
|
-
|
|
18269
|
+
}
|
|
18270
|
+
// Utility methods for querying audit logs
|
|
18271
|
+
async getAuditLogs(options = {}) {
|
|
18272
|
+
if (!this.auditResource) return [];
|
|
18273
|
+
try {
|
|
18274
|
+
const {
|
|
18275
|
+
resourceName,
|
|
18276
|
+
operation,
|
|
18277
|
+
recordId,
|
|
18278
|
+
userId,
|
|
18279
|
+
partition,
|
|
18280
|
+
startDate,
|
|
18281
|
+
endDate,
|
|
18282
|
+
limit = 100,
|
|
18283
|
+
offset = 0
|
|
18284
|
+
} = options;
|
|
18285
|
+
const allAudits = await this.auditResource.getAll();
|
|
18286
|
+
let filtered = allAudits.filter((audit) => {
|
|
18287
|
+
if (resourceName && audit.resourceName !== resourceName) return false;
|
|
18288
|
+
if (operation && audit.operation !== operation) return false;
|
|
18289
|
+
if (recordId && audit.recordId !== recordId) return false;
|
|
18290
|
+
if (userId && audit.userId !== userId) return false;
|
|
18291
|
+
if (partition && audit.partition !== partition) return false;
|
|
18292
|
+
if (startDate && new Date(audit.timestamp) < new Date(startDate)) return false;
|
|
18293
|
+
if (endDate && new Date(audit.timestamp) > new Date(endDate)) return false;
|
|
18294
|
+
return true;
|
|
18295
|
+
});
|
|
18296
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
18297
|
+
const deserialized = filtered.slice(offset, offset + limit).map((audit) => ({
|
|
18298
|
+
...audit,
|
|
18299
|
+
oldData: audit.oldData ? JSON.parse(audit.oldData) : null,
|
|
18300
|
+
newData: audit.newData ? JSON.parse(audit.newData) : null,
|
|
18301
|
+
partitionValues: audit.partitionValues ? JSON.parse(audit.partitionValues) : null,
|
|
18302
|
+
metadata: audit.metadata ? JSON.parse(audit.metadata) : null
|
|
18303
|
+
}));
|
|
18304
|
+
return deserialized;
|
|
18305
|
+
} catch (error) {
|
|
18306
|
+
console.error("Failed to get audit logs:", error);
|
|
18307
|
+
if (error && error.stack) console.error(error.stack);
|
|
18308
|
+
return [];
|
|
18309
|
+
}
|
|
18310
|
+
}
|
|
18311
|
+
async getRecordHistory(resourceName, recordId) {
|
|
18312
|
+
return this.getAuditLogs({
|
|
18313
|
+
resourceName,
|
|
18314
|
+
recordId,
|
|
18315
|
+
limit: 1e3
|
|
18316
|
+
});
|
|
18317
|
+
}
|
|
18318
|
+
async getPartitionHistory(resourceName, partitionName, partitionValues) {
|
|
18319
|
+
return this.getAuditLogs({
|
|
18320
|
+
resourceName,
|
|
18321
|
+
partition: partitionName,
|
|
18322
|
+
limit: 1e3
|
|
18323
|
+
});
|
|
18324
|
+
}
|
|
18325
|
+
async getAuditStats(options = {}) {
|
|
18326
|
+
const {
|
|
18327
|
+
resourceName,
|
|
18328
|
+
startDate,
|
|
18329
|
+
endDate
|
|
18330
|
+
} = options;
|
|
18331
|
+
const allAudits = await this.getAuditLogs({
|
|
18332
|
+
resourceName,
|
|
18333
|
+
startDate,
|
|
18334
|
+
endDate,
|
|
18335
|
+
limit: 1e4
|
|
18336
|
+
});
|
|
18337
|
+
const stats = {
|
|
18338
|
+
total: allAudits.length,
|
|
18339
|
+
byOperation: {},
|
|
18340
|
+
byResource: {},
|
|
18341
|
+
byPartition: {},
|
|
18342
|
+
byUser: {},
|
|
18343
|
+
timeline: {}
|
|
18344
|
+
};
|
|
18345
|
+
for (const audit of allAudits) {
|
|
18346
|
+
stats.byOperation[audit.operation] = (stats.byOperation[audit.operation] || 0) + 1;
|
|
18347
|
+
stats.byResource[audit.resourceName] = (stats.byResource[audit.resourceName] || 0) + 1;
|
|
18348
|
+
if (audit.partition) {
|
|
18349
|
+
stats.byPartition[audit.partition] = (stats.byPartition[audit.partition] || 0) + 1;
|
|
18350
|
+
}
|
|
18351
|
+
stats.byUser[audit.userId] = (stats.byUser[audit.userId] || 0) + 1;
|
|
18352
|
+
const day = audit.timestamp.split("T")[0];
|
|
18353
|
+
stats.timeline[day] = (stats.timeline[day] || 0) + 1;
|
|
18354
|
+
}
|
|
18355
|
+
return stats;
|
|
18356
|
+
}
|
|
18357
|
+
}
|
|
18358
|
+
|
|
18359
|
+
class CachePlugin extends Plugin {
|
|
18360
|
+
constructor(options = {}) {
|
|
18361
|
+
super(options);
|
|
18362
|
+
this.driver = options.driver;
|
|
18363
|
+
this.config = {
|
|
18364
|
+
enabled: options.enabled !== false,
|
|
18365
|
+
includePartitions: options.includePartitions !== false,
|
|
18366
|
+
...options
|
|
18367
|
+
};
|
|
18368
|
+
}
|
|
18369
|
+
async setup(database) {
|
|
18370
|
+
if (!this.config.enabled) {
|
|
18371
|
+
return;
|
|
18372
|
+
}
|
|
18373
|
+
await super.setup(database);
|
|
18374
|
+
}
|
|
18375
|
+
async onSetup() {
|
|
18376
|
+
if (this.config.driver) {
|
|
18377
|
+
this.driver = this.config.driver;
|
|
18378
|
+
} else if (this.config.driverType === "memory") {
|
|
18379
|
+
this.driver = new MemoryCache(this.config.memoryOptions || {});
|
|
18380
|
+
} else {
|
|
18381
|
+
this.driver = new S3Cache(this.config.s3Options || {});
|
|
18382
|
+
}
|
|
18383
|
+
this.installDatabaseProxy();
|
|
18384
|
+
this.installResourceHooks();
|
|
18385
|
+
}
|
|
18386
|
+
async onStart() {
|
|
18387
|
+
}
|
|
18388
|
+
async onStop() {
|
|
18389
|
+
}
|
|
18390
|
+
installDatabaseProxy() {
|
|
18391
|
+
if (this.database._cacheProxyInstalled) {
|
|
18392
|
+
return;
|
|
18393
|
+
}
|
|
18394
|
+
const installResourceHooks = this.installResourceHooks.bind(this);
|
|
18395
|
+
this.database._originalCreateResourceForCache = this.database.createResource;
|
|
18396
|
+
this.database.createResource = async function(...args) {
|
|
18397
|
+
const resource = await this._originalCreateResourceForCache(...args);
|
|
18398
|
+
installResourceHooks(resource);
|
|
18399
|
+
return resource;
|
|
18400
|
+
};
|
|
18401
|
+
this.database._cacheProxyInstalled = true;
|
|
18402
|
+
}
|
|
18403
|
+
installResourceHooks() {
|
|
18404
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18405
|
+
this.installResourceHooksForResource(resource);
|
|
18406
|
+
}
|
|
18407
|
+
}
|
|
18408
|
+
installResourceHooksForResource(resource) {
|
|
18409
|
+
if (!this.driver) return;
|
|
18410
|
+
resource.cache = this.driver;
|
|
18411
|
+
resource.cacheKeyFor = async (options = {}) => {
|
|
18412
|
+
const { action, params = {}, partition, partitionValues } = options;
|
|
18413
|
+
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
18414
|
+
};
|
|
18415
|
+
resource._originalCount = resource.count;
|
|
18416
|
+
resource._originalListIds = resource.listIds;
|
|
18417
|
+
resource._originalGetMany = resource.getMany;
|
|
18418
|
+
resource._originalGetAll = resource.getAll;
|
|
18419
|
+
resource._originalPage = resource.page;
|
|
18420
|
+
resource._originalList = resource.list;
|
|
18421
|
+
resource.count = async function(options = {}) {
|
|
18422
|
+
const { partition, partitionValues } = options;
|
|
18423
|
+
const key = await resource.cacheKeyFor({
|
|
18424
|
+
action: "count",
|
|
18425
|
+
partition,
|
|
18426
|
+
partitionValues
|
|
18427
|
+
});
|
|
18428
|
+
try {
|
|
18429
|
+
const cached = await resource.cache.get(key);
|
|
18430
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18431
|
+
} catch (err) {
|
|
18432
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18433
|
+
}
|
|
18434
|
+
const result = await resource._originalCount(options);
|
|
18435
|
+
await resource.cache.set(key, result);
|
|
18436
|
+
return result;
|
|
18437
|
+
};
|
|
18438
|
+
resource.listIds = async function(options = {}) {
|
|
18439
|
+
const { partition, partitionValues } = options;
|
|
18440
|
+
const key = await resource.cacheKeyFor({
|
|
18441
|
+
action: "listIds",
|
|
18442
|
+
partition,
|
|
18443
|
+
partitionValues
|
|
18444
|
+
});
|
|
18445
|
+
try {
|
|
18446
|
+
const cached = await resource.cache.get(key);
|
|
18447
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18448
|
+
} catch (err) {
|
|
18449
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18450
|
+
}
|
|
18451
|
+
const result = await resource._originalListIds(options);
|
|
18452
|
+
await resource.cache.set(key, result);
|
|
18453
|
+
return result;
|
|
18454
|
+
};
|
|
18455
|
+
resource.getMany = async function(ids) {
|
|
18456
|
+
const key = await resource.cacheKeyFor({
|
|
18457
|
+
action: "getMany",
|
|
18458
|
+
params: { ids }
|
|
18459
|
+
});
|
|
18460
|
+
try {
|
|
18461
|
+
const cached = await resource.cache.get(key);
|
|
18462
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18463
|
+
} catch (err) {
|
|
18464
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18465
|
+
}
|
|
18466
|
+
const result = await resource._originalGetMany(ids);
|
|
18467
|
+
await resource.cache.set(key, result);
|
|
18468
|
+
return result;
|
|
18469
|
+
};
|
|
18470
|
+
resource.getAll = async function() {
|
|
18471
|
+
const key = await resource.cacheKeyFor({ action: "getAll" });
|
|
18472
|
+
try {
|
|
18473
|
+
const cached = await resource.cache.get(key);
|
|
18474
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18475
|
+
} catch (err) {
|
|
18476
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18477
|
+
}
|
|
18478
|
+
const result = await resource._originalGetAll();
|
|
18479
|
+
await resource.cache.set(key, result);
|
|
18480
|
+
return result;
|
|
18481
|
+
};
|
|
18482
|
+
resource.page = async function({ offset, size, partition, partitionValues } = {}) {
|
|
18483
|
+
const key = await resource.cacheKeyFor({
|
|
18484
|
+
action: "page",
|
|
18485
|
+
params: { offset, size },
|
|
18486
|
+
partition,
|
|
18487
|
+
partitionValues
|
|
18488
|
+
});
|
|
18489
|
+
try {
|
|
18490
|
+
const cached = await resource.cache.get(key);
|
|
18491
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18492
|
+
} catch (err) {
|
|
18493
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18494
|
+
}
|
|
18495
|
+
const result = await resource._originalPage({ offset, size, partition, partitionValues });
|
|
18496
|
+
await resource.cache.set(key, result);
|
|
18497
|
+
return result;
|
|
18498
|
+
};
|
|
18499
|
+
resource.list = async function(options = {}) {
|
|
18500
|
+
const { partition, partitionValues } = options;
|
|
18501
|
+
const key = await resource.cacheKeyFor({
|
|
18502
|
+
action: "list",
|
|
18503
|
+
partition,
|
|
18504
|
+
partitionValues
|
|
18505
|
+
});
|
|
18506
|
+
try {
|
|
18507
|
+
const cached = await resource.cache.get(key);
|
|
18508
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18509
|
+
} catch (err) {
|
|
18510
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18511
|
+
}
|
|
18512
|
+
const result = await resource._originalList(options);
|
|
18513
|
+
await resource.cache.set(key, result);
|
|
18514
|
+
return result;
|
|
18515
|
+
};
|
|
18516
|
+
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
18517
|
+
const [data] = args;
|
|
18518
|
+
await this.clearCacheForResource(resource, data);
|
|
18519
|
+
return result;
|
|
18520
|
+
});
|
|
18521
|
+
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
18522
|
+
const [id, data] = args;
|
|
18523
|
+
await this.clearCacheForResource(resource, { id, ...data });
|
|
18524
|
+
return result;
|
|
18525
|
+
});
|
|
18526
|
+
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
18527
|
+
const [id] = args;
|
|
18528
|
+
let data = { id };
|
|
18529
|
+
if (typeof resource.get === "function") {
|
|
18530
|
+
try {
|
|
18531
|
+
const full = await resource.get(id);
|
|
18532
|
+
if (full) data = full;
|
|
18533
|
+
} catch {
|
|
18534
|
+
}
|
|
18535
|
+
}
|
|
18536
|
+
await this.clearCacheForResource(resource, data);
|
|
18537
|
+
return result;
|
|
18538
|
+
});
|
|
18539
|
+
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
18540
|
+
const [ids] = args;
|
|
18541
|
+
for (const id of ids) {
|
|
18542
|
+
let data = { id };
|
|
18543
|
+
if (typeof resource.get === "function") {
|
|
18544
|
+
try {
|
|
18545
|
+
const full = await resource.get(id);
|
|
18546
|
+
if (full) data = full;
|
|
18547
|
+
} catch {
|
|
18548
|
+
}
|
|
18549
|
+
}
|
|
18550
|
+
await this.clearCacheForResource(resource, data);
|
|
18551
|
+
}
|
|
18552
|
+
return result;
|
|
18553
|
+
});
|
|
18554
|
+
}
|
|
18555
|
+
async clearCacheForResource(resource, data) {
|
|
18556
|
+
if (!resource.cache) return;
|
|
18557
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
18558
|
+
await resource.cache.clear(keyPrefix);
|
|
18559
|
+
if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
|
|
18560
|
+
const partitionValues = this.getPartitionValues(data, resource);
|
|
18561
|
+
for (const [partitionName, values] of Object.entries(partitionValues)) {
|
|
18562
|
+
if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
|
|
18563
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
18564
|
+
await resource.cache.clear(partitionKeyPrefix);
|
|
18565
|
+
}
|
|
18566
|
+
}
|
|
18567
|
+
}
|
|
18568
|
+
}
|
|
18569
|
+
async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
|
|
18570
|
+
const keyParts = [
|
|
18571
|
+
`resource=${resource.name}`,
|
|
18572
|
+
`action=${action}`
|
|
18573
|
+
];
|
|
18574
|
+
if (partition && partitionValues && Object.keys(partitionValues).length > 0) {
|
|
18575
|
+
keyParts.push(`partition:${partition}`);
|
|
18576
|
+
for (const [field, value] of Object.entries(partitionValues)) {
|
|
18577
|
+
if (value !== null && value !== void 0) {
|
|
18578
|
+
keyParts.push(`${field}:${value}`);
|
|
18579
|
+
}
|
|
18580
|
+
}
|
|
18581
|
+
}
|
|
18582
|
+
if (Object.keys(params).length > 0) {
|
|
18583
|
+
const paramsHash = await this.hashParams(params);
|
|
18584
|
+
keyParts.push(paramsHash);
|
|
18585
|
+
}
|
|
18586
|
+
return join(...keyParts) + ".json.gz";
|
|
18587
|
+
}
|
|
18588
|
+
async hashParams(params) {
|
|
18589
|
+
const sortedParams = Object.keys(params).sort().map((key) => `${key}:${params[key]}`).join("|") || "empty";
|
|
18590
|
+
return await sha256(sortedParams);
|
|
18591
|
+
}
|
|
18592
|
+
// Utility methods
|
|
18593
|
+
async getCacheStats() {
|
|
18594
|
+
if (!this.driver) return null;
|
|
18595
|
+
return {
|
|
18596
|
+
size: await this.driver.size(),
|
|
18597
|
+
keys: await this.driver.keys(),
|
|
18598
|
+
driver: this.driver.constructor.name
|
|
18599
|
+
};
|
|
18600
|
+
}
|
|
18601
|
+
async clearAllCache() {
|
|
18602
|
+
if (!this.driver) return;
|
|
18603
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18604
|
+
if (resource.cache) {
|
|
18605
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
18606
|
+
await resource.cache.clear(keyPrefix);
|
|
18607
|
+
}
|
|
18608
|
+
}
|
|
18609
|
+
}
|
|
18610
|
+
async warmCache(resourceName, options = {}) {
|
|
18611
|
+
const resource = this.database.resources[resourceName];
|
|
18612
|
+
if (!resource) {
|
|
18613
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
18614
|
+
}
|
|
18615
|
+
const { includePartitions = true } = options;
|
|
18616
|
+
await resource.getAll();
|
|
18617
|
+
if (includePartitions && resource.config.partitions) {
|
|
18618
|
+
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
18619
|
+
if (partitionDef.fields) {
|
|
18620
|
+
const allRecords = await resource.getAll();
|
|
18621
|
+
const recordsArray = Array.isArray(allRecords) ? allRecords : [];
|
|
18622
|
+
const partitionValues = /* @__PURE__ */ new Set();
|
|
18623
|
+
for (const record of recordsArray.slice(0, 10)) {
|
|
18624
|
+
const values = this.getPartitionValues(record, resource);
|
|
18625
|
+
if (values[partitionName]) {
|
|
18626
|
+
partitionValues.add(JSON.stringify(values[partitionName]));
|
|
18627
|
+
}
|
|
18628
|
+
}
|
|
18629
|
+
for (const partitionValueStr of partitionValues) {
|
|
18630
|
+
const partitionValues2 = JSON.parse(partitionValueStr);
|
|
18631
|
+
await resource.list({ partition: partitionName, partitionValues: partitionValues2 });
|
|
18632
|
+
}
|
|
18633
|
+
}
|
|
18634
|
+
}
|
|
18635
|
+
}
|
|
18636
|
+
}
|
|
18637
|
+
}
|
|
18638
|
+
|
|
18639
|
+
const CostsPlugin = {
|
|
18640
|
+
async setup(db) {
|
|
18641
|
+
if (!db || !db.client) {
|
|
18642
|
+
return;
|
|
18643
|
+
}
|
|
18644
|
+
this.client = db.client;
|
|
18645
|
+
this.map = {
|
|
18646
|
+
PutObjectCommand: "put",
|
|
18647
|
+
GetObjectCommand: "get",
|
|
18648
|
+
HeadObjectCommand: "head",
|
|
18649
|
+
DeleteObjectCommand: "delete",
|
|
18650
|
+
DeleteObjectsCommand: "delete",
|
|
18651
|
+
ListObjectsV2Command: "list"
|
|
18652
|
+
};
|
|
18653
|
+
this.costs = {
|
|
18654
|
+
total: 0,
|
|
18655
|
+
prices: {
|
|
18656
|
+
put: 5e-3 / 1e3,
|
|
18657
|
+
copy: 5e-3 / 1e3,
|
|
18658
|
+
list: 5e-3 / 1e3,
|
|
18659
|
+
post: 5e-3 / 1e3,
|
|
18660
|
+
get: 4e-4 / 1e3,
|
|
18661
|
+
select: 4e-4 / 1e3,
|
|
18662
|
+
delete: 4e-4 / 1e3,
|
|
18663
|
+
head: 4e-4 / 1e3
|
|
18664
|
+
},
|
|
18665
|
+
requests: {
|
|
18666
|
+
total: 0,
|
|
18667
|
+
put: 0,
|
|
18668
|
+
post: 0,
|
|
18669
|
+
copy: 0,
|
|
18670
|
+
list: 0,
|
|
18671
|
+
get: 0,
|
|
18672
|
+
select: 0,
|
|
18673
|
+
delete: 0,
|
|
18674
|
+
head: 0
|
|
18675
|
+
},
|
|
18676
|
+
events: {
|
|
18677
|
+
total: 0,
|
|
18678
|
+
PutObjectCommand: 0,
|
|
18679
|
+
GetObjectCommand: 0,
|
|
18680
|
+
HeadObjectCommand: 0,
|
|
18681
|
+
DeleteObjectCommand: 0,
|
|
18682
|
+
DeleteObjectsCommand: 0,
|
|
18683
|
+
ListObjectsV2Command: 0
|
|
18684
|
+
}
|
|
18685
|
+
};
|
|
18686
|
+
this.client.costs = JSON.parse(JSON.stringify(this.costs));
|
|
18687
|
+
},
|
|
18688
|
+
async start() {
|
|
18689
|
+
if (this.client) {
|
|
18690
|
+
this.client.on("command.response", (name) => this.addRequest(name, this.map[name]));
|
|
18691
|
+
this.client.on("command.error", (name) => this.addRequest(name, this.map[name]));
|
|
18692
|
+
}
|
|
18693
|
+
},
|
|
18694
|
+
addRequest(name, method) {
|
|
18695
|
+
if (!method) return;
|
|
18696
|
+
this.costs.events[name]++;
|
|
18697
|
+
this.costs.events.total++;
|
|
18698
|
+
this.costs.requests.total++;
|
|
18699
|
+
this.costs.requests[method]++;
|
|
18700
|
+
this.costs.total += this.costs.prices[method];
|
|
18701
|
+
if (this.client && this.client.costs) {
|
|
18702
|
+
this.client.costs.events[name]++;
|
|
18703
|
+
this.client.costs.events.total++;
|
|
18704
|
+
this.client.costs.requests.total++;
|
|
18705
|
+
this.client.costs.requests[method]++;
|
|
18706
|
+
this.client.costs.total += this.client.costs.prices[method];
|
|
18707
|
+
}
|
|
18708
|
+
}
|
|
18709
|
+
};
|
|
18710
|
+
|
|
18711
|
+
class FullTextPlugin extends Plugin {
|
|
18712
|
+
constructor(options = {}) {
|
|
18713
|
+
super();
|
|
18714
|
+
this.indexResource = null;
|
|
18715
|
+
this.config = {
|
|
18716
|
+
enabled: options.enabled !== false,
|
|
18717
|
+
minWordLength: options.minWordLength || 3,
|
|
18718
|
+
maxResults: options.maxResults || 100,
|
|
18719
|
+
...options
|
|
18720
|
+
};
|
|
18721
|
+
this.indexes = /* @__PURE__ */ new Map();
|
|
18722
|
+
}
|
|
18723
|
+
async setup(database) {
|
|
18724
|
+
this.database = database;
|
|
18725
|
+
if (!this.config.enabled) return;
|
|
18726
|
+
try {
|
|
18727
|
+
this.indexResource = await database.createResource({
|
|
18728
|
+
name: "fulltext_indexes",
|
|
18729
|
+
attributes: {
|
|
18730
|
+
id: "string|required",
|
|
18731
|
+
resourceName: "string|required",
|
|
18732
|
+
fieldName: "string|required",
|
|
18733
|
+
word: "string|required",
|
|
18734
|
+
recordIds: "json|required",
|
|
18735
|
+
// Array of record IDs containing this word
|
|
18736
|
+
count: "number|required",
|
|
18737
|
+
lastUpdated: "string|required"
|
|
18738
|
+
}
|
|
18739
|
+
});
|
|
18740
|
+
} catch (error) {
|
|
18741
|
+
this.indexResource = database.resources.fulltext_indexes;
|
|
18742
|
+
}
|
|
18743
|
+
await this.loadIndexes();
|
|
18744
|
+
this.installIndexingHooks();
|
|
18745
|
+
}
|
|
18746
|
+
async start() {
|
|
18747
|
+
}
|
|
18748
|
+
async stop() {
|
|
18749
|
+
await this.saveIndexes();
|
|
18750
|
+
}
|
|
18751
|
+
async loadIndexes() {
|
|
18752
|
+
if (!this.indexResource) return;
|
|
18753
|
+
try {
|
|
18754
|
+
const allIndexes = await this.indexResource.getAll();
|
|
18755
|
+
for (const indexRecord of allIndexes) {
|
|
18756
|
+
const key = `${indexRecord.resourceName}:${indexRecord.fieldName}:${indexRecord.word}`;
|
|
18757
|
+
this.indexes.set(key, {
|
|
18758
|
+
recordIds: indexRecord.recordIds || [],
|
|
18759
|
+
count: indexRecord.count || 0
|
|
18760
|
+
});
|
|
18761
|
+
}
|
|
18762
|
+
} catch (error) {
|
|
18763
|
+
console.warn("Failed to load existing indexes:", error.message);
|
|
18764
|
+
}
|
|
18765
|
+
}
|
|
18766
|
+
async saveIndexes() {
|
|
18767
|
+
if (!this.indexResource) return;
|
|
18768
|
+
try {
|
|
18769
|
+
const existingIndexes = await this.indexResource.getAll();
|
|
18770
|
+
for (const index of existingIndexes) {
|
|
18771
|
+
await this.indexResource.delete(index.id);
|
|
18772
|
+
}
|
|
18773
|
+
for (const [key, data] of this.indexes.entries()) {
|
|
18774
|
+
const [resourceName, fieldName, word] = key.split(":");
|
|
18775
|
+
await this.indexResource.insert({
|
|
18776
|
+
id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18777
|
+
resourceName,
|
|
18778
|
+
fieldName,
|
|
18779
|
+
word,
|
|
18780
|
+
recordIds: data.recordIds,
|
|
18781
|
+
count: data.count,
|
|
18782
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
18783
|
+
});
|
|
18784
|
+
}
|
|
18785
|
+
} catch (error) {
|
|
18786
|
+
console.error("Failed to save indexes:", error);
|
|
18787
|
+
}
|
|
18788
|
+
}
|
|
18789
|
+
installIndexingHooks() {
|
|
18790
|
+
if (!this.database.plugins) {
|
|
18791
|
+
this.database.plugins = {};
|
|
18792
|
+
}
|
|
18793
|
+
this.database.plugins.fulltext = this;
|
|
18794
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18795
|
+
if (resource.name === "fulltext_indexes") continue;
|
|
18796
|
+
this.installResourceHooks(resource);
|
|
18797
|
+
}
|
|
18798
|
+
if (!this.database._fulltextProxyInstalled) {
|
|
18799
|
+
this.database._previousCreateResourceForFullText = this.database.createResource;
|
|
18800
|
+
this.database.createResource = async function(...args) {
|
|
18801
|
+
const resource = await this._previousCreateResourceForFullText(...args);
|
|
18802
|
+
if (this.plugins?.fulltext && resource.name !== "fulltext_indexes") {
|
|
18803
|
+
this.plugins.fulltext.installResourceHooks(resource);
|
|
18804
|
+
}
|
|
18805
|
+
return resource;
|
|
18806
|
+
};
|
|
18807
|
+
this.database._fulltextProxyInstalled = true;
|
|
18808
|
+
}
|
|
18809
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18810
|
+
if (resource.name !== "fulltext_indexes") {
|
|
18811
|
+
this.installResourceHooks(resource);
|
|
18812
|
+
}
|
|
18813
|
+
}
|
|
18814
|
+
}
|
|
18815
|
+
installResourceHooks(resource) {
|
|
18816
|
+
resource._insert = resource.insert;
|
|
18817
|
+
resource._update = resource.update;
|
|
18818
|
+
resource._delete = resource.delete;
|
|
18819
|
+
resource._deleteMany = resource.deleteMany;
|
|
18820
|
+
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
18821
|
+
const [data] = args;
|
|
18822
|
+
this.indexRecord(resource.name, result.id, data).catch(console.error);
|
|
18823
|
+
return result;
|
|
18824
|
+
});
|
|
18825
|
+
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
18826
|
+
const [id, data] = args;
|
|
18827
|
+
this.removeRecordFromIndex(resource.name, id).catch(console.error);
|
|
18828
|
+
this.indexRecord(resource.name, id, result).catch(console.error);
|
|
18829
|
+
return result;
|
|
18830
|
+
});
|
|
18831
|
+
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
18832
|
+
const [id] = args;
|
|
18833
|
+
this.removeRecordFromIndex(resource.name, id).catch(console.error);
|
|
18834
|
+
return result;
|
|
18835
|
+
});
|
|
18836
|
+
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
18837
|
+
const [ids] = args;
|
|
18838
|
+
for (const id of ids) {
|
|
18839
|
+
this.removeRecordFromIndex(resource.name, id).catch(console.error);
|
|
18840
|
+
}
|
|
18841
|
+
return result;
|
|
18842
|
+
});
|
|
18843
|
+
}
|
|
18844
|
+
async indexRecord(resourceName, recordId, data) {
|
|
18845
|
+
const indexedFields = this.getIndexedFields(resourceName);
|
|
18846
|
+
if (!indexedFields || indexedFields.length === 0) return;
|
|
18847
|
+
for (const fieldName of indexedFields) {
|
|
18848
|
+
const fieldValue = this.getFieldValue(data, fieldName);
|
|
18849
|
+
if (!fieldValue) continue;
|
|
18850
|
+
const words = this.tokenize(fieldValue);
|
|
18851
|
+
for (const word of words) {
|
|
18852
|
+
if (word.length < this.config.minWordLength) continue;
|
|
18853
|
+
const key = `${resourceName}:${fieldName}:${word.toLowerCase()}`;
|
|
18854
|
+
const existing = this.indexes.get(key) || { recordIds: [], count: 0 };
|
|
18855
|
+
if (!existing.recordIds.includes(recordId)) {
|
|
18856
|
+
existing.recordIds.push(recordId);
|
|
18857
|
+
existing.count = existing.recordIds.length;
|
|
18858
|
+
}
|
|
18859
|
+
this.indexes.set(key, existing);
|
|
18860
|
+
}
|
|
18861
|
+
}
|
|
18862
|
+
}
|
|
18863
|
+
async removeRecordFromIndex(resourceName, recordId) {
|
|
18864
|
+
for (const [key, data] of this.indexes.entries()) {
|
|
18865
|
+
if (key.startsWith(`${resourceName}:`)) {
|
|
18866
|
+
const index = data.recordIds.indexOf(recordId);
|
|
18867
|
+
if (index > -1) {
|
|
18868
|
+
data.recordIds.splice(index, 1);
|
|
18869
|
+
data.count = data.recordIds.length;
|
|
18870
|
+
if (data.recordIds.length === 0) {
|
|
18871
|
+
this.indexes.delete(key);
|
|
18872
|
+
} else {
|
|
18873
|
+
this.indexes.set(key, data);
|
|
18874
|
+
}
|
|
18875
|
+
}
|
|
18876
|
+
}
|
|
18877
|
+
}
|
|
18878
|
+
}
|
|
18879
|
+
getFieldValue(data, fieldPath) {
|
|
18880
|
+
if (!fieldPath.includes(".")) {
|
|
18881
|
+
return data[fieldPath];
|
|
18882
|
+
}
|
|
18883
|
+
const keys = fieldPath.split(".");
|
|
18884
|
+
let value = data;
|
|
18885
|
+
for (const key of keys) {
|
|
18886
|
+
if (value && typeof value === "object" && key in value) {
|
|
18887
|
+
value = value[key];
|
|
18888
|
+
} else {
|
|
18889
|
+
return null;
|
|
18890
|
+
}
|
|
18891
|
+
}
|
|
18892
|
+
return value;
|
|
18893
|
+
}
|
|
18894
|
+
tokenize(text) {
|
|
18895
|
+
if (!text) return [];
|
|
18896
|
+
const str = String(text).toLowerCase();
|
|
18897
|
+
return str.replace(/[^\w\s\u00C0-\u017F]/g, " ").split(/\s+/).filter((word) => word.length > 0);
|
|
18898
|
+
}
|
|
18899
|
+
getIndexedFields(resourceName) {
|
|
18900
|
+
if (this.config.fields) {
|
|
18901
|
+
return this.config.fields;
|
|
18902
|
+
}
|
|
18903
|
+
const fieldMappings = {
|
|
18904
|
+
users: ["name", "email"],
|
|
18905
|
+
products: ["name", "description"],
|
|
18906
|
+
articles: ["title", "content"]
|
|
18907
|
+
// Add more mappings as needed
|
|
18908
|
+
};
|
|
18909
|
+
return fieldMappings[resourceName] || [];
|
|
18910
|
+
}
|
|
18911
|
+
// Main search method
|
|
18912
|
+
async search(resourceName, query, options = {}) {
|
|
18913
|
+
const {
|
|
18914
|
+
fields = null,
|
|
18915
|
+
// Specific fields to search in
|
|
18916
|
+
limit = this.config.maxResults,
|
|
18917
|
+
offset = 0,
|
|
18918
|
+
exactMatch = false
|
|
18919
|
+
} = options;
|
|
18920
|
+
if (!query || query.trim().length === 0) {
|
|
18921
|
+
return [];
|
|
18922
|
+
}
|
|
18923
|
+
const searchWords = this.tokenize(query);
|
|
18924
|
+
const results = /* @__PURE__ */ new Map();
|
|
18925
|
+
const searchFields = fields || this.getIndexedFields(resourceName);
|
|
18926
|
+
if (searchFields.length === 0) {
|
|
18927
|
+
return [];
|
|
18928
|
+
}
|
|
18929
|
+
for (const word of searchWords) {
|
|
18930
|
+
if (word.length < this.config.minWordLength) continue;
|
|
18931
|
+
for (const fieldName of searchFields) {
|
|
18932
|
+
if (exactMatch) {
|
|
18933
|
+
const key = `${resourceName}:${fieldName}:${word.toLowerCase()}`;
|
|
18934
|
+
const indexData = this.indexes.get(key);
|
|
18935
|
+
if (indexData) {
|
|
18936
|
+
for (const recordId of indexData.recordIds) {
|
|
18937
|
+
const currentScore = results.get(recordId) || 0;
|
|
18938
|
+
results.set(recordId, currentScore + 1);
|
|
18939
|
+
}
|
|
18940
|
+
}
|
|
18941
|
+
} else {
|
|
18942
|
+
for (const [key, indexData] of this.indexes.entries()) {
|
|
18943
|
+
if (key.startsWith(`${resourceName}:${fieldName}:${word.toLowerCase()}`)) {
|
|
18944
|
+
for (const recordId of indexData.recordIds) {
|
|
18945
|
+
const currentScore = results.get(recordId) || 0;
|
|
18946
|
+
results.set(recordId, currentScore + 1);
|
|
18947
|
+
}
|
|
18948
|
+
}
|
|
18949
|
+
}
|
|
18950
|
+
}
|
|
18951
|
+
}
|
|
18952
|
+
}
|
|
18953
|
+
const sortedResults = Array.from(results.entries()).map(([recordId, score]) => ({ recordId, score })).sort((a, b) => b.score - a.score).slice(offset, offset + limit);
|
|
18954
|
+
return sortedResults;
|
|
18955
|
+
}
|
|
18956
|
+
// Search and return full records
|
|
18957
|
+
async searchRecords(resourceName, query, options = {}) {
|
|
18958
|
+
const searchResults = await this.search(resourceName, query, options);
|
|
18959
|
+
if (searchResults.length === 0) {
|
|
18960
|
+
return [];
|
|
18961
|
+
}
|
|
18962
|
+
const resource = this.database.resources[resourceName];
|
|
18963
|
+
if (!resource) {
|
|
18964
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
18965
|
+
}
|
|
18966
|
+
const recordIds = searchResults.map((result) => result.recordId);
|
|
18967
|
+
const records = await resource.getMany(recordIds);
|
|
18968
|
+
return records.map((record) => {
|
|
18969
|
+
const searchResult = searchResults.find((sr) => sr.recordId === record.id);
|
|
18970
|
+
return {
|
|
18971
|
+
...record,
|
|
18972
|
+
_searchScore: searchResult ? searchResult.score : 0
|
|
18973
|
+
};
|
|
18974
|
+
}).sort((a, b) => b._searchScore - a._searchScore);
|
|
18975
|
+
}
|
|
18976
|
+
// Utility methods
|
|
18977
|
+
async rebuildIndex(resourceName) {
|
|
18978
|
+
const resource = this.database.resources[resourceName];
|
|
18979
|
+
if (!resource) {
|
|
18980
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
18981
|
+
}
|
|
18982
|
+
for (const [key] of this.indexes.entries()) {
|
|
18983
|
+
if (key.startsWith(`${resourceName}:`)) {
|
|
18984
|
+
this.indexes.delete(key);
|
|
18985
|
+
}
|
|
18986
|
+
}
|
|
18987
|
+
const allRecords = await resource.getAll();
|
|
18988
|
+
const batchSize = 100;
|
|
18989
|
+
for (let i = 0; i < allRecords.length; i += batchSize) {
|
|
18990
|
+
const batch = allRecords.slice(i, i + batchSize);
|
|
18991
|
+
for (const record of batch) {
|
|
18992
|
+
await this.indexRecord(resourceName, record.id, record);
|
|
18993
|
+
}
|
|
18994
|
+
}
|
|
18995
|
+
await this.saveIndexes();
|
|
18996
|
+
}
|
|
18997
|
+
async getIndexStats() {
|
|
18998
|
+
const stats = {
|
|
18999
|
+
totalIndexes: this.indexes.size,
|
|
19000
|
+
resources: {},
|
|
19001
|
+
totalWords: 0
|
|
19002
|
+
};
|
|
19003
|
+
for (const [key, data] of this.indexes.entries()) {
|
|
19004
|
+
const [resourceName, fieldName] = key.split(":");
|
|
19005
|
+
if (!stats.resources[resourceName]) {
|
|
19006
|
+
stats.resources[resourceName] = {
|
|
19007
|
+
fields: {},
|
|
19008
|
+
totalRecords: /* @__PURE__ */ new Set(),
|
|
19009
|
+
totalWords: 0
|
|
19010
|
+
};
|
|
19011
|
+
}
|
|
19012
|
+
if (!stats.resources[resourceName].fields[fieldName]) {
|
|
19013
|
+
stats.resources[resourceName].fields[fieldName] = {
|
|
19014
|
+
words: 0,
|
|
19015
|
+
totalOccurrences: 0
|
|
19016
|
+
};
|
|
19017
|
+
}
|
|
19018
|
+
stats.resources[resourceName].fields[fieldName].words++;
|
|
19019
|
+
stats.resources[resourceName].fields[fieldName].totalOccurrences += data.count;
|
|
19020
|
+
stats.resources[resourceName].totalWords++;
|
|
19021
|
+
for (const recordId of data.recordIds) {
|
|
19022
|
+
stats.resources[resourceName].totalRecords.add(recordId);
|
|
19023
|
+
}
|
|
19024
|
+
stats.totalWords++;
|
|
19025
|
+
}
|
|
19026
|
+
for (const resourceName in stats.resources) {
|
|
19027
|
+
stats.resources[resourceName].totalRecords = stats.resources[resourceName].totalRecords.size;
|
|
19028
|
+
}
|
|
19029
|
+
return stats;
|
|
19030
|
+
}
|
|
19031
|
+
async rebuildAllIndexes({ timeout } = {}) {
|
|
19032
|
+
if (timeout) {
|
|
19033
|
+
return Promise.race([
|
|
19034
|
+
this._rebuildAllIndexesInternal(),
|
|
19035
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout")), timeout))
|
|
19036
|
+
]);
|
|
19037
|
+
}
|
|
19038
|
+
return this._rebuildAllIndexesInternal();
|
|
19039
|
+
}
|
|
19040
|
+
async _rebuildAllIndexesInternal() {
|
|
19041
|
+
const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "fulltext_indexes");
|
|
19042
|
+
for (const resourceName of resourceNames) {
|
|
19043
|
+
try {
|
|
19044
|
+
await this.rebuildIndex(resourceName);
|
|
19045
|
+
} catch (error) {
|
|
19046
|
+
console.warn(`Failed to rebuild index for resource ${resourceName}:`, error.message);
|
|
19047
|
+
}
|
|
19048
|
+
}
|
|
19049
|
+
}
|
|
19050
|
+
async clearIndex(resourceName) {
|
|
19051
|
+
for (const [key] of this.indexes.entries()) {
|
|
19052
|
+
if (key.startsWith(`${resourceName}:`)) {
|
|
19053
|
+
this.indexes.delete(key);
|
|
19054
|
+
}
|
|
19055
|
+
}
|
|
19056
|
+
await this.saveIndexes();
|
|
19057
|
+
}
|
|
19058
|
+
async clearAllIndexes() {
|
|
19059
|
+
this.indexes.clear();
|
|
19060
|
+
await this.saveIndexes();
|
|
19061
|
+
}
|
|
19062
|
+
}
|
|
19063
|
+
|
|
19064
|
+
class MetricsPlugin extends Plugin {
|
|
19065
|
+
constructor(options = {}) {
|
|
19066
|
+
super();
|
|
19067
|
+
this.config = {
|
|
19068
|
+
enabled: options.enabled !== false,
|
|
19069
|
+
collectPerformance: options.collectPerformance !== false,
|
|
19070
|
+
collectErrors: options.collectErrors !== false,
|
|
19071
|
+
collectUsage: options.collectUsage !== false,
|
|
19072
|
+
retentionDays: options.retentionDays || 30,
|
|
19073
|
+
flushInterval: options.flushInterval || 6e4,
|
|
19074
|
+
// 1 minute
|
|
19075
|
+
...options
|
|
19076
|
+
};
|
|
19077
|
+
this.metrics = {
|
|
19078
|
+
operations: {
|
|
19079
|
+
insert: { count: 0, totalTime: 0, errors: 0 },
|
|
19080
|
+
update: { count: 0, totalTime: 0, errors: 0 },
|
|
19081
|
+
delete: { count: 0, totalTime: 0, errors: 0 },
|
|
19082
|
+
get: { count: 0, totalTime: 0, errors: 0 },
|
|
19083
|
+
list: { count: 0, totalTime: 0, errors: 0 },
|
|
19084
|
+
count: { count: 0, totalTime: 0, errors: 0 }
|
|
19085
|
+
},
|
|
19086
|
+
resources: {},
|
|
19087
|
+
errors: [],
|
|
19088
|
+
performance: [],
|
|
19089
|
+
startTime: (/* @__PURE__ */ new Date()).toISOString()
|
|
19090
|
+
};
|
|
19091
|
+
this.flushTimer = null;
|
|
19092
|
+
}
|
|
19093
|
+
async setup(database) {
|
|
19094
|
+
this.database = database;
|
|
19095
|
+
if (!this.config.enabled || process.env.NODE_ENV === "test") return;
|
|
19096
|
+
try {
|
|
19097
|
+
this.metricsResource = await database.createResource({
|
|
19098
|
+
name: "metrics",
|
|
19099
|
+
attributes: {
|
|
19100
|
+
id: "string|required",
|
|
19101
|
+
type: "string|required",
|
|
19102
|
+
// 'operation', 'error', 'performance'
|
|
19103
|
+
resourceName: "string",
|
|
19104
|
+
operation: "string",
|
|
19105
|
+
count: "number|required",
|
|
19106
|
+
totalTime: "number|required",
|
|
19107
|
+
errors: "number|required",
|
|
19108
|
+
avgTime: "number|required",
|
|
19109
|
+
timestamp: "string|required",
|
|
19110
|
+
metadata: "json"
|
|
19111
|
+
}
|
|
19112
|
+
});
|
|
19113
|
+
this.errorsResource = await database.createResource({
|
|
19114
|
+
name: "error_logs",
|
|
19115
|
+
attributes: {
|
|
19116
|
+
id: "string|required",
|
|
19117
|
+
resourceName: "string|required",
|
|
19118
|
+
operation: "string|required",
|
|
19119
|
+
error: "string|required",
|
|
19120
|
+
timestamp: "string|required",
|
|
19121
|
+
metadata: "json"
|
|
19122
|
+
}
|
|
19123
|
+
});
|
|
19124
|
+
this.performanceResource = await database.createResource({
|
|
19125
|
+
name: "performance_logs",
|
|
19126
|
+
attributes: {
|
|
19127
|
+
id: "string|required",
|
|
19128
|
+
resourceName: "string|required",
|
|
19129
|
+
operation: "string|required",
|
|
19130
|
+
duration: "number|required",
|
|
19131
|
+
timestamp: "string|required",
|
|
19132
|
+
metadata: "json"
|
|
19133
|
+
}
|
|
19134
|
+
});
|
|
19135
|
+
} catch (error) {
|
|
19136
|
+
this.metricsResource = database.resources.metrics;
|
|
19137
|
+
this.errorsResource = database.resources.error_logs;
|
|
19138
|
+
this.performanceResource = database.resources.performance_logs;
|
|
19139
|
+
}
|
|
19140
|
+
this.installMetricsHooks();
|
|
19141
|
+
if (process.env.NODE_ENV !== "test") {
|
|
19142
|
+
this.startFlushTimer();
|
|
19143
|
+
}
|
|
19144
|
+
}
|
|
19145
|
+
async start() {
|
|
19146
|
+
}
|
|
19147
|
+
async stop() {
|
|
19148
|
+
if (this.flushTimer) {
|
|
19149
|
+
clearInterval(this.flushTimer);
|
|
19150
|
+
this.flushTimer = null;
|
|
19151
|
+
}
|
|
19152
|
+
if (process.env.NODE_ENV !== "test") {
|
|
19153
|
+
await this.flushMetrics();
|
|
19154
|
+
}
|
|
19155
|
+
}
|
|
19156
|
+
installMetricsHooks() {
|
|
19157
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
19158
|
+
if (["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
|
|
19159
|
+
continue;
|
|
19160
|
+
}
|
|
19161
|
+
this.installResourceHooks(resource);
|
|
19162
|
+
}
|
|
19163
|
+
this.database._createResource = this.database.createResource;
|
|
19164
|
+
this.database.createResource = async function(...args) {
|
|
19165
|
+
const resource = await this._createResource(...args);
|
|
19166
|
+
if (this.plugins?.metrics && !["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
|
|
19167
|
+
this.plugins.metrics.installResourceHooks(resource);
|
|
19168
|
+
}
|
|
19169
|
+
return resource;
|
|
19170
|
+
};
|
|
19171
|
+
}
|
|
19172
|
+
installResourceHooks(resource) {
|
|
19173
|
+
resource._insert = resource.insert;
|
|
19174
|
+
resource._update = resource.update;
|
|
19175
|
+
resource._delete = resource.delete;
|
|
19176
|
+
resource._deleteMany = resource.deleteMany;
|
|
19177
|
+
resource._get = resource.get;
|
|
19178
|
+
resource._getMany = resource.getMany;
|
|
19179
|
+
resource._getAll = resource.getAll;
|
|
19180
|
+
resource._list = resource.list;
|
|
19181
|
+
resource._listIds = resource.listIds;
|
|
19182
|
+
resource._count = resource.count;
|
|
19183
|
+
resource._page = resource.page;
|
|
19184
|
+
resource.insert = async function(...args) {
|
|
19185
|
+
const startTime = Date.now();
|
|
19186
|
+
try {
|
|
19187
|
+
const result = await resource._insert(...args);
|
|
19188
|
+
this.recordOperation(resource.name, "insert", Date.now() - startTime, false);
|
|
19189
|
+
return result;
|
|
19190
|
+
} catch (error) {
|
|
19191
|
+
this.recordOperation(resource.name, "insert", Date.now() - startTime, true);
|
|
19192
|
+
this.recordError(resource.name, "insert", error);
|
|
19193
|
+
throw error;
|
|
19194
|
+
}
|
|
19195
|
+
}.bind(this);
|
|
19196
|
+
resource.update = async function(...args) {
|
|
19197
|
+
const startTime = Date.now();
|
|
19198
|
+
try {
|
|
19199
|
+
const result = await resource._update(...args);
|
|
19200
|
+
this.recordOperation(resource.name, "update", Date.now() - startTime, false);
|
|
19201
|
+
return result;
|
|
19202
|
+
} catch (error) {
|
|
19203
|
+
this.recordOperation(resource.name, "update", Date.now() - startTime, true);
|
|
19204
|
+
this.recordError(resource.name, "update", error);
|
|
19205
|
+
throw error;
|
|
19206
|
+
}
|
|
19207
|
+
}.bind(this);
|
|
19208
|
+
resource.delete = async function(...args) {
|
|
19209
|
+
const startTime = Date.now();
|
|
19210
|
+
try {
|
|
19211
|
+
const result = await resource._delete(...args);
|
|
19212
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, false);
|
|
19213
|
+
return result;
|
|
19214
|
+
} catch (error) {
|
|
19215
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, true);
|
|
19216
|
+
this.recordError(resource.name, "delete", error);
|
|
19217
|
+
throw error;
|
|
19218
|
+
}
|
|
19219
|
+
}.bind(this);
|
|
19220
|
+
resource.deleteMany = async function(...args) {
|
|
19221
|
+
const startTime = Date.now();
|
|
19222
|
+
try {
|
|
19223
|
+
const result = await resource._deleteMany(...args);
|
|
19224
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, false);
|
|
19225
|
+
return result;
|
|
19226
|
+
} catch (error) {
|
|
19227
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, true);
|
|
19228
|
+
this.recordError(resource.name, "delete", error);
|
|
19229
|
+
throw error;
|
|
19230
|
+
}
|
|
19231
|
+
}.bind(this);
|
|
19232
|
+
resource.get = async function(...args) {
|
|
19233
|
+
const startTime = Date.now();
|
|
19234
|
+
try {
|
|
19235
|
+
const result = await resource._get(...args);
|
|
19236
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, false);
|
|
19237
|
+
return result;
|
|
19238
|
+
} catch (error) {
|
|
19239
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, true);
|
|
19240
|
+
this.recordError(resource.name, "get", error);
|
|
19241
|
+
throw error;
|
|
19242
|
+
}
|
|
19243
|
+
}.bind(this);
|
|
19244
|
+
resource.getMany = async function(...args) {
|
|
19245
|
+
const startTime = Date.now();
|
|
19246
|
+
try {
|
|
19247
|
+
const result = await resource._getMany(...args);
|
|
19248
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, false);
|
|
19249
|
+
return result;
|
|
19250
|
+
} catch (error) {
|
|
19251
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, true);
|
|
19252
|
+
this.recordError(resource.name, "get", error);
|
|
19253
|
+
throw error;
|
|
19254
|
+
}
|
|
19255
|
+
}.bind(this);
|
|
19256
|
+
resource.getAll = async function(...args) {
|
|
19257
|
+
const startTime = Date.now();
|
|
19258
|
+
try {
|
|
19259
|
+
const result = await resource._getAll(...args);
|
|
19260
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19261
|
+
return result;
|
|
19262
|
+
} catch (error) {
|
|
19263
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19264
|
+
this.recordError(resource.name, "list", error);
|
|
19265
|
+
throw error;
|
|
19266
|
+
}
|
|
19267
|
+
}.bind(this);
|
|
19268
|
+
resource.list = async function(...args) {
|
|
19269
|
+
const startTime = Date.now();
|
|
19270
|
+
try {
|
|
19271
|
+
const result = await resource._list(...args);
|
|
19272
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19273
|
+
return result;
|
|
19274
|
+
} catch (error) {
|
|
19275
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19276
|
+
this.recordError(resource.name, "list", error);
|
|
19277
|
+
throw error;
|
|
19278
|
+
}
|
|
19279
|
+
}.bind(this);
|
|
19280
|
+
resource.listIds = async function(...args) {
|
|
19281
|
+
const startTime = Date.now();
|
|
19282
|
+
try {
|
|
19283
|
+
const result = await resource._listIds(...args);
|
|
19284
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19285
|
+
return result;
|
|
19286
|
+
} catch (error) {
|
|
19287
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19288
|
+
this.recordError(resource.name, "list", error);
|
|
19289
|
+
throw error;
|
|
19290
|
+
}
|
|
19291
|
+
}.bind(this);
|
|
19292
|
+
resource.count = async function(...args) {
|
|
19293
|
+
const startTime = Date.now();
|
|
19294
|
+
try {
|
|
19295
|
+
const result = await resource._count(...args);
|
|
19296
|
+
this.recordOperation(resource.name, "count", Date.now() - startTime, false);
|
|
19297
|
+
return result;
|
|
19298
|
+
} catch (error) {
|
|
19299
|
+
this.recordOperation(resource.name, "count", Date.now() - startTime, true);
|
|
19300
|
+
this.recordError(resource.name, "count", error);
|
|
19301
|
+
throw error;
|
|
19302
|
+
}
|
|
19303
|
+
}.bind(this);
|
|
19304
|
+
resource.page = async function(...args) {
|
|
19305
|
+
const startTime = Date.now();
|
|
19306
|
+
try {
|
|
19307
|
+
const result = await resource._page(...args);
|
|
19308
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19309
|
+
return result;
|
|
19310
|
+
} catch (error) {
|
|
19311
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19312
|
+
this.recordError(resource.name, "list", error);
|
|
19313
|
+
throw error;
|
|
19314
|
+
}
|
|
19315
|
+
}.bind(this);
|
|
19316
|
+
}
|
|
19317
|
+
recordOperation(resourceName, operation, duration, isError) {
|
|
19318
|
+
if (this.metrics.operations[operation]) {
|
|
19319
|
+
this.metrics.operations[operation].count++;
|
|
19320
|
+
this.metrics.operations[operation].totalTime += duration;
|
|
19321
|
+
if (isError) {
|
|
19322
|
+
this.metrics.operations[operation].errors++;
|
|
19323
|
+
}
|
|
19324
|
+
}
|
|
19325
|
+
if (!this.metrics.resources[resourceName]) {
|
|
19326
|
+
this.metrics.resources[resourceName] = {
|
|
19327
|
+
insert: { count: 0, totalTime: 0, errors: 0 },
|
|
19328
|
+
update: { count: 0, totalTime: 0, errors: 0 },
|
|
19329
|
+
delete: { count: 0, totalTime: 0, errors: 0 },
|
|
19330
|
+
get: { count: 0, totalTime: 0, errors: 0 },
|
|
19331
|
+
list: { count: 0, totalTime: 0, errors: 0 },
|
|
19332
|
+
count: { count: 0, totalTime: 0, errors: 0 }
|
|
19333
|
+
};
|
|
19334
|
+
}
|
|
19335
|
+
if (this.metrics.resources[resourceName][operation]) {
|
|
19336
|
+
this.metrics.resources[resourceName][operation].count++;
|
|
19337
|
+
this.metrics.resources[resourceName][operation].totalTime += duration;
|
|
19338
|
+
if (isError) {
|
|
19339
|
+
this.metrics.resources[resourceName][operation].errors++;
|
|
19340
|
+
}
|
|
19341
|
+
}
|
|
19342
|
+
if (this.config.collectPerformance) {
|
|
19343
|
+
this.metrics.performance.push({
|
|
19344
|
+
resourceName,
|
|
19345
|
+
operation,
|
|
19346
|
+
duration,
|
|
19347
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
19348
|
+
});
|
|
19349
|
+
}
|
|
19350
|
+
}
|
|
19351
|
+
recordError(resourceName, operation, error) {
|
|
19352
|
+
if (!this.config.collectErrors) return;
|
|
19353
|
+
this.metrics.errors.push({
|
|
19354
|
+
resourceName,
|
|
19355
|
+
operation,
|
|
19356
|
+
error: error.message,
|
|
19357
|
+
stack: error.stack,
|
|
19358
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
19359
|
+
});
|
|
19360
|
+
}
|
|
19361
|
+
startFlushTimer() {
|
|
19362
|
+
if (this.flushTimer) {
|
|
19363
|
+
clearInterval(this.flushTimer);
|
|
19364
|
+
}
|
|
19365
|
+
if (this.config.flushInterval > 0) {
|
|
19366
|
+
this.flushTimer = setInterval(() => {
|
|
19367
|
+
this.flushMetrics().catch(console.error);
|
|
19368
|
+
}, this.config.flushInterval);
|
|
19369
|
+
}
|
|
19370
|
+
}
|
|
19371
|
+
async flushMetrics() {
|
|
19372
|
+
if (!this.metricsResource) return;
|
|
19373
|
+
try {
|
|
19374
|
+
const metadata = process.env.NODE_ENV === "test" ? {} : { global: "true" };
|
|
19375
|
+
const perfMetadata = process.env.NODE_ENV === "test" ? {} : { perf: "true" };
|
|
19376
|
+
const errorMetadata = process.env.NODE_ENV === "test" ? {} : { error: "true" };
|
|
19377
|
+
const resourceMetadata = process.env.NODE_ENV === "test" ? {} : { resource: "true" };
|
|
19378
|
+
for (const [operation, data] of Object.entries(this.metrics.operations)) {
|
|
19379
|
+
if (data.count > 0) {
|
|
19380
|
+
await this.metricsResource.insert({
|
|
19381
|
+
id: `metrics-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19382
|
+
type: "operation",
|
|
19383
|
+
resourceName: "global",
|
|
19384
|
+
operation,
|
|
19385
|
+
count: data.count,
|
|
19386
|
+
totalTime: data.totalTime,
|
|
19387
|
+
errors: data.errors,
|
|
19388
|
+
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
19389
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19390
|
+
metadata
|
|
19391
|
+
});
|
|
19392
|
+
}
|
|
19393
|
+
}
|
|
19394
|
+
for (const [resourceName, operations] of Object.entries(this.metrics.resources)) {
|
|
19395
|
+
for (const [operation, data] of Object.entries(operations)) {
|
|
19396
|
+
if (data.count > 0) {
|
|
19397
|
+
await this.metricsResource.insert({
|
|
19398
|
+
id: `metrics-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19399
|
+
type: "operation",
|
|
19400
|
+
resourceName,
|
|
19401
|
+
operation,
|
|
19402
|
+
count: data.count,
|
|
19403
|
+
totalTime: data.totalTime,
|
|
19404
|
+
errors: data.errors,
|
|
19405
|
+
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
19406
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19407
|
+
metadata: resourceMetadata
|
|
19408
|
+
});
|
|
19409
|
+
}
|
|
19410
|
+
}
|
|
19411
|
+
}
|
|
19412
|
+
if (this.config.collectPerformance && this.metrics.performance.length > 0) {
|
|
19413
|
+
for (const perf of this.metrics.performance) {
|
|
19414
|
+
await this.performanceResource.insert({
|
|
19415
|
+
id: `perf-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19416
|
+
resourceName: perf.resourceName,
|
|
19417
|
+
operation: perf.operation,
|
|
19418
|
+
duration: perf.duration,
|
|
19419
|
+
timestamp: perf.timestamp,
|
|
19420
|
+
metadata: perfMetadata
|
|
19421
|
+
});
|
|
19422
|
+
}
|
|
19423
|
+
}
|
|
19424
|
+
if (this.config.collectErrors && this.metrics.errors.length > 0) {
|
|
19425
|
+
for (const error of this.metrics.errors) {
|
|
19426
|
+
await this.errorsResource.insert({
|
|
19427
|
+
id: `error-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19428
|
+
resourceName: error.resourceName,
|
|
19429
|
+
operation: error.operation,
|
|
19430
|
+
error: error.error,
|
|
19431
|
+
stack: error.stack,
|
|
19432
|
+
timestamp: error.timestamp,
|
|
19433
|
+
metadata: errorMetadata
|
|
19434
|
+
});
|
|
19435
|
+
}
|
|
19436
|
+
}
|
|
19437
|
+
this.resetMetrics();
|
|
19438
|
+
} catch (error) {
|
|
19439
|
+
console.error("Failed to flush metrics:", error);
|
|
19440
|
+
}
|
|
19441
|
+
}
|
|
19442
|
+
resetMetrics() {
|
|
19443
|
+
for (const operation of Object.keys(this.metrics.operations)) {
|
|
19444
|
+
this.metrics.operations[operation] = { count: 0, totalTime: 0, errors: 0 };
|
|
19445
|
+
}
|
|
19446
|
+
for (const resourceName of Object.keys(this.metrics.resources)) {
|
|
19447
|
+
for (const operation of Object.keys(this.metrics.resources[resourceName])) {
|
|
19448
|
+
this.metrics.resources[resourceName][operation] = { count: 0, totalTime: 0, errors: 0 };
|
|
19449
|
+
}
|
|
19450
|
+
}
|
|
19451
|
+
this.metrics.performance = [];
|
|
19452
|
+
this.metrics.errors = [];
|
|
19453
|
+
}
|
|
19454
|
+
// Utility methods
|
|
19455
|
+
async getMetrics(options = {}) {
|
|
19456
|
+
const {
|
|
19457
|
+
type = "operation",
|
|
19458
|
+
resourceName,
|
|
19459
|
+
operation,
|
|
19460
|
+
startDate,
|
|
19461
|
+
endDate,
|
|
19462
|
+
limit = 100,
|
|
19463
|
+
offset = 0
|
|
19464
|
+
} = options;
|
|
19465
|
+
if (!this.metricsResource) return [];
|
|
19466
|
+
const allMetrics = await this.metricsResource.getAll();
|
|
19467
|
+
let filtered = allMetrics.filter((metric) => {
|
|
19468
|
+
if (type && metric.type !== type) return false;
|
|
19469
|
+
if (resourceName && metric.resourceName !== resourceName) return false;
|
|
19470
|
+
if (operation && metric.operation !== operation) return false;
|
|
19471
|
+
if (startDate && new Date(metric.timestamp) < new Date(startDate)) return false;
|
|
19472
|
+
if (endDate && new Date(metric.timestamp) > new Date(endDate)) return false;
|
|
19473
|
+
return true;
|
|
19474
|
+
});
|
|
19475
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
19476
|
+
return filtered.slice(offset, offset + limit);
|
|
19477
|
+
}
|
|
19478
|
+
async getErrorLogs(options = {}) {
|
|
19479
|
+
if (!this.errorsResource) return [];
|
|
19480
|
+
const {
|
|
19481
|
+
resourceName,
|
|
19482
|
+
operation,
|
|
19483
|
+
startDate,
|
|
19484
|
+
endDate,
|
|
19485
|
+
limit = 100,
|
|
19486
|
+
offset = 0
|
|
19487
|
+
} = options;
|
|
19488
|
+
const allErrors = await this.errorsResource.getAll();
|
|
19489
|
+
let filtered = allErrors.filter((error) => {
|
|
19490
|
+
if (resourceName && error.resourceName !== resourceName) return false;
|
|
19491
|
+
if (operation && error.operation !== operation) return false;
|
|
19492
|
+
if (startDate && new Date(error.timestamp) < new Date(startDate)) return false;
|
|
19493
|
+
if (endDate && new Date(error.timestamp) > new Date(endDate)) return false;
|
|
19494
|
+
return true;
|
|
19495
|
+
});
|
|
19496
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
19497
|
+
return filtered.slice(offset, offset + limit);
|
|
19498
|
+
}
|
|
19499
|
+
async getPerformanceLogs(options = {}) {
|
|
19500
|
+
if (!this.performanceResource) return [];
|
|
19501
|
+
const {
|
|
19502
|
+
resourceName,
|
|
19503
|
+
operation,
|
|
19504
|
+
startDate,
|
|
19505
|
+
endDate,
|
|
19506
|
+
limit = 100,
|
|
19507
|
+
offset = 0
|
|
19508
|
+
} = options;
|
|
19509
|
+
const allPerformance = await this.performanceResource.getAll();
|
|
19510
|
+
let filtered = allPerformance.filter((perf) => {
|
|
19511
|
+
if (resourceName && perf.resourceName !== resourceName) return false;
|
|
19512
|
+
if (operation && perf.operation !== operation) return false;
|
|
19513
|
+
if (startDate && new Date(perf.timestamp) < new Date(startDate)) return false;
|
|
19514
|
+
if (endDate && new Date(perf.timestamp) > new Date(endDate)) return false;
|
|
19515
|
+
return true;
|
|
19516
|
+
});
|
|
19517
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
19518
|
+
return filtered.slice(offset, offset + limit);
|
|
19519
|
+
}
|
|
19520
|
+
async getStats() {
|
|
19521
|
+
const now = /* @__PURE__ */ new Date();
|
|
19522
|
+
const startDate = new Date(now.getTime() - 24 * 60 * 60 * 1e3);
|
|
19523
|
+
const [metrics, errors, performance] = await Promise.all([
|
|
19524
|
+
this.getMetrics({ startDate: startDate.toISOString() }),
|
|
19525
|
+
this.getErrorLogs({ startDate: startDate.toISOString() }),
|
|
19526
|
+
this.getPerformanceLogs({ startDate: startDate.toISOString() })
|
|
19527
|
+
]);
|
|
19528
|
+
const stats = {
|
|
19529
|
+
period: "24h",
|
|
19530
|
+
totalOperations: 0,
|
|
19531
|
+
totalErrors: errors.length,
|
|
19532
|
+
avgResponseTime: 0,
|
|
19533
|
+
operationsByType: {},
|
|
19534
|
+
resources: {},
|
|
19535
|
+
uptime: {
|
|
19536
|
+
startTime: this.metrics.startTime,
|
|
19537
|
+
duration: now.getTime() - new Date(this.metrics.startTime).getTime()
|
|
19538
|
+
}
|
|
19539
|
+
};
|
|
19540
|
+
for (const metric of metrics) {
|
|
19541
|
+
if (metric.type === "operation") {
|
|
19542
|
+
stats.totalOperations += metric.count;
|
|
19543
|
+
if (!stats.operationsByType[metric.operation]) {
|
|
19544
|
+
stats.operationsByType[metric.operation] = {
|
|
19545
|
+
count: 0,
|
|
19546
|
+
errors: 0,
|
|
19547
|
+
avgTime: 0
|
|
19548
|
+
};
|
|
19549
|
+
}
|
|
19550
|
+
stats.operationsByType[metric.operation].count += metric.count;
|
|
19551
|
+
stats.operationsByType[metric.operation].errors += metric.errors;
|
|
19552
|
+
const current = stats.operationsByType[metric.operation];
|
|
19553
|
+
const totalCount2 = current.count;
|
|
19554
|
+
const newAvg = (current.avgTime * (totalCount2 - metric.count) + metric.totalTime) / totalCount2;
|
|
19555
|
+
current.avgTime = newAvg;
|
|
19556
|
+
}
|
|
19557
|
+
}
|
|
19558
|
+
const totalTime = metrics.reduce((sum, m) => sum + m.totalTime, 0);
|
|
19559
|
+
const totalCount = metrics.reduce((sum, m) => sum + m.count, 0);
|
|
19560
|
+
stats.avgResponseTime = totalCount > 0 ? totalTime / totalCount : 0;
|
|
19561
|
+
return stats;
|
|
19562
|
+
}
|
|
19563
|
+
async cleanupOldData() {
|
|
19564
|
+
const cutoffDate = /* @__PURE__ */ new Date();
|
|
19565
|
+
cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
|
|
19566
|
+
if (this.metricsResource) {
|
|
19567
|
+
const oldMetrics = await this.getMetrics({ endDate: cutoffDate.toISOString() });
|
|
19568
|
+
for (const metric of oldMetrics) {
|
|
19569
|
+
await this.metricsResource.delete(metric.id);
|
|
19570
|
+
}
|
|
19571
|
+
}
|
|
19572
|
+
if (this.errorsResource) {
|
|
19573
|
+
const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
|
|
19574
|
+
for (const error of oldErrors) {
|
|
19575
|
+
await this.errorsResource.delete(error.id);
|
|
19576
|
+
}
|
|
19577
|
+
}
|
|
19578
|
+
if (this.performanceResource) {
|
|
19579
|
+
const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
|
|
19580
|
+
for (const perf of oldPerformance) {
|
|
19581
|
+
await this.performanceResource.delete(perf.id);
|
|
19582
|
+
}
|
|
19583
|
+
}
|
|
19584
|
+
console.log(`Cleaned up data older than ${this.config.retentionDays} days`);
|
|
19585
|
+
}
|
|
19586
|
+
}
|
|
19587
|
+
|
|
19588
|
+
class BaseReplicator extends EventEmitter {
|
|
19589
|
+
constructor(config = {}) {
|
|
19590
|
+
super();
|
|
19591
|
+
this.config = config;
|
|
19592
|
+
this.name = this.constructor.name;
|
|
19593
|
+
this.enabled = config.enabled !== false;
|
|
19594
|
+
}
|
|
19595
|
+
/**
|
|
19596
|
+
* Initialize the replicator
|
|
19597
|
+
* @param {Object} database - The s3db database instance
|
|
19598
|
+
* @returns {Promise<void>}
|
|
19599
|
+
*/
|
|
19600
|
+
async initialize(database) {
|
|
19601
|
+
this.database = database;
|
|
19602
|
+
this.emit("initialized", { replicator: this.name });
|
|
19603
|
+
}
|
|
19604
|
+
/**
|
|
19605
|
+
* Replicate data to the target
|
|
19606
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
19607
|
+
* @param {string} operation - Operation type (insert, update, delete)
|
|
19608
|
+
* @param {Object} data - The data to replicate
|
|
19609
|
+
* @param {string} id - Record ID
|
|
19610
|
+
* @returns {Promise<Object>} Replication result
|
|
19611
|
+
*/
|
|
19612
|
+
async replicate(resourceName, operation, data, id) {
|
|
19613
|
+
throw new Error(`replicate() method must be implemented by ${this.name}`);
|
|
19614
|
+
}
|
|
19615
|
+
/**
|
|
19616
|
+
* Replicate multiple records in batch
|
|
19617
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
19618
|
+
* @param {Array} records - Array of records to replicate
|
|
19619
|
+
* @returns {Promise<Object>} Batch replication result
|
|
19620
|
+
*/
|
|
19621
|
+
async replicateBatch(resourceName, records) {
|
|
19622
|
+
throw new Error(`replicateBatch() method must be implemented by ${this.name}`);
|
|
19623
|
+
}
|
|
19624
|
+
/**
|
|
19625
|
+
* Test the connection to the target
|
|
19626
|
+
* @returns {Promise<boolean>} True if connection is successful
|
|
19627
|
+
*/
|
|
19628
|
+
async testConnection() {
|
|
19629
|
+
throw new Error(`testConnection() method must be implemented by ${this.name}`);
|
|
19630
|
+
}
|
|
19631
|
+
/**
|
|
19632
|
+
* Get replicator status and statistics
|
|
19633
|
+
* @returns {Promise<Object>} Status information
|
|
19634
|
+
*/
|
|
19635
|
+
async getStatus() {
|
|
19636
|
+
return {
|
|
19637
|
+
name: this.name,
|
|
19638
|
+
enabled: this.enabled,
|
|
19639
|
+
config: this.config,
|
|
19640
|
+
connected: false
|
|
19641
|
+
};
|
|
19642
|
+
}
|
|
19643
|
+
/**
|
|
19644
|
+
* Cleanup resources
|
|
19645
|
+
* @returns {Promise<void>}
|
|
19646
|
+
*/
|
|
19647
|
+
async cleanup() {
|
|
19648
|
+
this.emit("cleanup", { replicator: this.name });
|
|
19649
|
+
}
|
|
19650
|
+
/**
|
|
19651
|
+
* Validate replicator configuration
|
|
19652
|
+
* @returns {Object} Validation result
|
|
19653
|
+
*/
|
|
19654
|
+
validateConfig() {
|
|
19655
|
+
return { isValid: true, errors: [] };
|
|
19656
|
+
}
|
|
19657
|
+
}
|
|
19658
|
+
|
|
19659
|
+
class S3dbReplicator extends BaseReplicator {
|
|
19660
|
+
constructor(config = {}, resources = []) {
|
|
19661
|
+
super(config);
|
|
19662
|
+
this.resources = resources;
|
|
19663
|
+
this.connectionString = config.connectionString;
|
|
19664
|
+
this.region = config.region;
|
|
19665
|
+
this.bucket = config.bucket;
|
|
19666
|
+
this.keyPrefix = config.keyPrefix;
|
|
19667
|
+
}
|
|
19668
|
+
validateConfig() {
|
|
19669
|
+
const errors = [];
|
|
19670
|
+
if (!this.connectionString && !this.bucket) {
|
|
19671
|
+
errors.push("Either connectionString or bucket must be provided");
|
|
19672
|
+
}
|
|
19673
|
+
return {
|
|
19674
|
+
isValid: errors.length === 0,
|
|
19675
|
+
errors
|
|
19676
|
+
};
|
|
19677
|
+
}
|
|
19678
|
+
async initialize(database) {
|
|
19679
|
+
await super.initialize(database);
|
|
19680
|
+
const targetConfig = {
|
|
19681
|
+
connectionString: this.connectionString,
|
|
19682
|
+
region: this.region,
|
|
19683
|
+
bucket: this.bucket,
|
|
19684
|
+
keyPrefix: this.keyPrefix,
|
|
19685
|
+
verbose: this.config.verbose || false
|
|
19686
|
+
};
|
|
19687
|
+
this.targetDatabase = new S3db(targetConfig);
|
|
19688
|
+
await this.targetDatabase.connect();
|
|
19689
|
+
this.emit("connected", {
|
|
19690
|
+
replicator: this.name,
|
|
19691
|
+
target: this.connectionString || this.bucket
|
|
19692
|
+
});
|
|
19693
|
+
}
|
|
19694
|
+
async replicate(resourceName, operation, data, id) {
|
|
19695
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19696
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19697
|
+
}
|
|
19698
|
+
try {
|
|
19699
|
+
let result;
|
|
19700
|
+
switch (operation) {
|
|
19701
|
+
case "insert":
|
|
19702
|
+
result = await this.targetDatabase.resources[resourceName]?.insert(data);
|
|
19703
|
+
break;
|
|
19704
|
+
case "update":
|
|
19705
|
+
result = await this.targetDatabase.resources[resourceName]?.update(id, data);
|
|
19706
|
+
break;
|
|
19707
|
+
case "delete":
|
|
19708
|
+
result = await this.targetDatabase.resources[resourceName]?.delete(id);
|
|
19709
|
+
break;
|
|
19710
|
+
default:
|
|
19711
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
19712
|
+
}
|
|
19713
|
+
this.emit("replicated", {
|
|
19714
|
+
replicator: this.name,
|
|
19715
|
+
resourceName,
|
|
19716
|
+
operation,
|
|
19717
|
+
id,
|
|
19718
|
+
success: true
|
|
19719
|
+
});
|
|
19720
|
+
return { success: true, result };
|
|
19721
|
+
} catch (error) {
|
|
19722
|
+
this.emit("replication_error", {
|
|
19723
|
+
replicator: this.name,
|
|
19724
|
+
resourceName,
|
|
19725
|
+
operation,
|
|
19726
|
+
id,
|
|
19727
|
+
error: error.message
|
|
19728
|
+
});
|
|
19729
|
+
return { success: false, error: error.message };
|
|
19730
|
+
}
|
|
19731
|
+
}
|
|
19732
|
+
async replicateBatch(resourceName, records) {
|
|
19733
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19734
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19735
|
+
}
|
|
19736
|
+
try {
|
|
19737
|
+
const results = [];
|
|
19738
|
+
const errors = [];
|
|
19739
|
+
for (const record of records) {
|
|
19740
|
+
try {
|
|
19741
|
+
const result = await this.replicate(
|
|
19742
|
+
resourceName,
|
|
19743
|
+
record.operation,
|
|
19744
|
+
record.data,
|
|
19745
|
+
record.id
|
|
19746
|
+
);
|
|
19747
|
+
results.push(result);
|
|
19748
|
+
} catch (error) {
|
|
19749
|
+
errors.push({ id: record.id, error: error.message });
|
|
19750
|
+
}
|
|
19751
|
+
}
|
|
19752
|
+
this.emit("batch_replicated", {
|
|
19753
|
+
replicator: this.name,
|
|
19754
|
+
resourceName,
|
|
19755
|
+
total: records.length,
|
|
19756
|
+
successful: results.filter((r) => r.success).length,
|
|
19757
|
+
errors: errors.length
|
|
19758
|
+
});
|
|
19759
|
+
return {
|
|
19760
|
+
success: errors.length === 0,
|
|
19761
|
+
results,
|
|
19762
|
+
errors,
|
|
19763
|
+
total: records.length
|
|
19764
|
+
};
|
|
19765
|
+
} catch (error) {
|
|
19766
|
+
this.emit("batch_replication_error", {
|
|
19767
|
+
replicator: this.name,
|
|
19768
|
+
resourceName,
|
|
19769
|
+
error: error.message
|
|
19770
|
+
});
|
|
19771
|
+
return { success: false, error: error.message };
|
|
19772
|
+
}
|
|
19773
|
+
}
|
|
19774
|
+
async testConnection() {
|
|
19775
|
+
try {
|
|
19776
|
+
if (!this.targetDatabase) {
|
|
19777
|
+
await this.initialize(this.database);
|
|
19778
|
+
}
|
|
19779
|
+
await this.targetDatabase.listResources();
|
|
19780
|
+
return true;
|
|
19781
|
+
} catch (error) {
|
|
19782
|
+
this.emit("connection_error", {
|
|
19783
|
+
replicator: this.name,
|
|
19784
|
+
error: error.message
|
|
19785
|
+
});
|
|
19786
|
+
return false;
|
|
19787
|
+
}
|
|
19788
|
+
}
|
|
19789
|
+
async getStatus() {
|
|
19790
|
+
const baseStatus = await super.getStatus();
|
|
19791
|
+
return {
|
|
19792
|
+
...baseStatus,
|
|
19793
|
+
connected: !!this.targetDatabase,
|
|
19794
|
+
targetDatabase: this.connectionString || this.bucket,
|
|
19795
|
+
resources: this.resources,
|
|
19796
|
+
totalReplications: this.listenerCount("replicated"),
|
|
19797
|
+
totalErrors: this.listenerCount("replication_error")
|
|
19798
|
+
};
|
|
19799
|
+
}
|
|
19800
|
+
async cleanup() {
|
|
19801
|
+
if (this.targetDatabase) {
|
|
19802
|
+
this.targetDatabase.removeAllListeners();
|
|
19803
|
+
}
|
|
19804
|
+
await super.cleanup();
|
|
19805
|
+
}
|
|
19806
|
+
shouldReplicateResource(resourceName) {
|
|
19807
|
+
return this.resources.length === 0 || this.resources.includes(resourceName);
|
|
19808
|
+
}
|
|
19809
|
+
}
|
|
19810
|
+
|
|
19811
|
+
class SqsReplicator extends BaseReplicator {
|
|
19812
|
+
constructor(config = {}, resources = []) {
|
|
19813
|
+
super(config);
|
|
19814
|
+
this.resources = resources;
|
|
19815
|
+
this.queueUrl = config.queueUrl;
|
|
19816
|
+
this.queues = config.queues || {};
|
|
19817
|
+
this.defaultQueueUrl = config.defaultQueueUrl;
|
|
19818
|
+
this.region = config.region || "us-east-1";
|
|
19819
|
+
this.sqsClient = null;
|
|
19820
|
+
this.messageGroupId = config.messageGroupId;
|
|
19821
|
+
this.deduplicationId = config.deduplicationId;
|
|
19822
|
+
}
|
|
19823
|
+
validateConfig() {
|
|
19824
|
+
const errors = [];
|
|
19825
|
+
if (!this.queueUrl && Object.keys(this.queues).length === 0 && !this.defaultQueueUrl) {
|
|
19826
|
+
errors.push("Either queueUrl, queues object, or defaultQueueUrl must be provided");
|
|
19827
|
+
}
|
|
19828
|
+
return {
|
|
19829
|
+
isValid: errors.length === 0,
|
|
19830
|
+
errors
|
|
19831
|
+
};
|
|
19832
|
+
}
|
|
19833
|
+
/**
|
|
19834
|
+
* Get the appropriate queue URL for a resource
|
|
19835
|
+
*/
|
|
19836
|
+
getQueueUrlForResource(resourceName) {
|
|
19837
|
+
if (this.queues[resourceName]) {
|
|
19838
|
+
return this.queues[resourceName];
|
|
19839
|
+
}
|
|
19840
|
+
if (this.queueUrl) {
|
|
19841
|
+
return this.queueUrl;
|
|
19842
|
+
}
|
|
19843
|
+
if (this.defaultQueueUrl) {
|
|
19844
|
+
return this.defaultQueueUrl;
|
|
19845
|
+
}
|
|
19846
|
+
throw new Error(`No queue URL found for resource '${resourceName}'`);
|
|
19847
|
+
}
|
|
19848
|
+
/**
|
|
19849
|
+
* Create standardized message structure
|
|
19850
|
+
*/
|
|
19851
|
+
createMessage(resourceName, operation, data, id, beforeData = null) {
|
|
19852
|
+
const baseMessage = {
|
|
19853
|
+
resource: resourceName,
|
|
19854
|
+
action: operation,
|
|
19855
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19856
|
+
source: "s3db-replication"
|
|
19857
|
+
};
|
|
19858
|
+
switch (operation) {
|
|
19859
|
+
case "insert":
|
|
19860
|
+
return {
|
|
19861
|
+
...baseMessage,
|
|
19862
|
+
data
|
|
19863
|
+
};
|
|
19864
|
+
case "update":
|
|
19865
|
+
return {
|
|
19866
|
+
...baseMessage,
|
|
19867
|
+
before: beforeData,
|
|
19868
|
+
data
|
|
19869
|
+
};
|
|
19870
|
+
case "delete":
|
|
19871
|
+
return {
|
|
19872
|
+
...baseMessage,
|
|
19873
|
+
data
|
|
19874
|
+
};
|
|
19875
|
+
default:
|
|
19876
|
+
return {
|
|
19877
|
+
...baseMessage,
|
|
19878
|
+
data
|
|
19879
|
+
};
|
|
19880
|
+
}
|
|
19881
|
+
}
|
|
19882
|
+
async initialize(database) {
|
|
19883
|
+
await super.initialize(database);
|
|
19884
|
+
try {
|
|
19885
|
+
const { SQSClient, SendMessageCommand, SendMessageBatchCommand } = await import('@aws-sdk/client-sqs');
|
|
19886
|
+
this.sqsClient = new SQSClient({
|
|
19887
|
+
region: this.region,
|
|
19888
|
+
credentials: this.config.credentials
|
|
19889
|
+
});
|
|
19890
|
+
this.emit("initialized", {
|
|
19891
|
+
replicator: this.name,
|
|
19892
|
+
queueUrl: this.queueUrl,
|
|
19893
|
+
queues: this.queues,
|
|
19894
|
+
defaultQueueUrl: this.defaultQueueUrl
|
|
19895
|
+
});
|
|
19896
|
+
} catch (error) {
|
|
19897
|
+
this.emit("initialization_error", {
|
|
19898
|
+
replicator: this.name,
|
|
19899
|
+
error: error.message
|
|
19900
|
+
});
|
|
19901
|
+
throw error;
|
|
19902
|
+
}
|
|
19903
|
+
}
|
|
19904
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
19905
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19906
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19907
|
+
}
|
|
19908
|
+
try {
|
|
19909
|
+
const { SendMessageCommand } = await import('@aws-sdk/client-sqs');
|
|
19910
|
+
const queueUrl = this.getQueueUrlForResource(resourceName);
|
|
19911
|
+
const message = this.createMessage(resourceName, operation, data, id, beforeData);
|
|
19912
|
+
const command = new SendMessageCommand({
|
|
19913
|
+
QueueUrl: queueUrl,
|
|
19914
|
+
MessageBody: JSON.stringify(message),
|
|
19915
|
+
MessageGroupId: this.messageGroupId,
|
|
19916
|
+
MessageDeduplicationId: this.deduplicationId ? `${resourceName}:${operation}:${id}` : void 0
|
|
19917
|
+
});
|
|
19918
|
+
const result = await this.sqsClient.send(command);
|
|
19919
|
+
this.emit("replicated", {
|
|
19920
|
+
replicator: this.name,
|
|
19921
|
+
resourceName,
|
|
19922
|
+
operation,
|
|
19923
|
+
id,
|
|
19924
|
+
queueUrl,
|
|
19925
|
+
messageId: result.MessageId,
|
|
19926
|
+
success: true
|
|
19927
|
+
});
|
|
19928
|
+
return { success: true, messageId: result.MessageId, queueUrl };
|
|
19929
|
+
} catch (error) {
|
|
19930
|
+
this.emit("replication_error", {
|
|
19931
|
+
replicator: this.name,
|
|
19932
|
+
resourceName,
|
|
19933
|
+
operation,
|
|
19934
|
+
id,
|
|
19935
|
+
error: error.message
|
|
19936
|
+
});
|
|
19937
|
+
return { success: false, error: error.message };
|
|
19938
|
+
}
|
|
19939
|
+
}
|
|
19940
|
+
async replicateBatch(resourceName, records) {
|
|
19941
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19942
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19943
|
+
}
|
|
19944
|
+
try {
|
|
19945
|
+
const { SendMessageBatchCommand } = await import('@aws-sdk/client-sqs');
|
|
19946
|
+
const queueUrl = this.getQueueUrlForResource(resourceName);
|
|
19947
|
+
const batchSize = 10;
|
|
19948
|
+
const batches = [];
|
|
19949
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
19950
|
+
batches.push(records.slice(i, i + batchSize));
|
|
19951
|
+
}
|
|
19952
|
+
const results = [];
|
|
19953
|
+
const errors = [];
|
|
19954
|
+
for (const batch of batches) {
|
|
19955
|
+
try {
|
|
19956
|
+
const entries = batch.map((record, index) => ({
|
|
19957
|
+
Id: `${record.id}-${index}`,
|
|
19958
|
+
MessageBody: JSON.stringify(this.createMessage(
|
|
19959
|
+
resourceName,
|
|
19960
|
+
record.operation,
|
|
19961
|
+
record.data,
|
|
19962
|
+
record.id,
|
|
19963
|
+
record.beforeData
|
|
19964
|
+
)),
|
|
19965
|
+
MessageGroupId: this.messageGroupId,
|
|
19966
|
+
MessageDeduplicationId: this.deduplicationId ? `${resourceName}:${record.operation}:${record.id}` : void 0
|
|
19967
|
+
}));
|
|
19968
|
+
const command = new SendMessageBatchCommand({
|
|
19969
|
+
QueueUrl: queueUrl,
|
|
19970
|
+
Entries: entries
|
|
19971
|
+
});
|
|
19972
|
+
const result = await this.sqsClient.send(command);
|
|
19973
|
+
results.push(result);
|
|
19974
|
+
} catch (error) {
|
|
19975
|
+
errors.push({ batch: batch.length, error: error.message });
|
|
19976
|
+
}
|
|
19977
|
+
}
|
|
19978
|
+
this.emit("batch_replicated", {
|
|
19979
|
+
replicator: this.name,
|
|
19980
|
+
resourceName,
|
|
19981
|
+
queueUrl,
|
|
19982
|
+
total: records.length,
|
|
19983
|
+
successful: results.length,
|
|
19984
|
+
errors: errors.length
|
|
19985
|
+
});
|
|
19986
|
+
return {
|
|
19987
|
+
success: errors.length === 0,
|
|
19988
|
+
results,
|
|
19989
|
+
errors,
|
|
19990
|
+
total: records.length,
|
|
19991
|
+
queueUrl
|
|
19992
|
+
};
|
|
19993
|
+
} catch (error) {
|
|
19994
|
+
this.emit("batch_replication_error", {
|
|
19995
|
+
replicator: this.name,
|
|
19996
|
+
resourceName,
|
|
19997
|
+
error: error.message
|
|
19998
|
+
});
|
|
19999
|
+
return { success: false, error: error.message };
|
|
20000
|
+
}
|
|
20001
|
+
}
|
|
20002
|
+
async testConnection() {
|
|
20003
|
+
try {
|
|
20004
|
+
if (!this.sqsClient) {
|
|
20005
|
+
await this.initialize(this.database);
|
|
20006
|
+
}
|
|
20007
|
+
const { GetQueueAttributesCommand } = await import('@aws-sdk/client-sqs');
|
|
20008
|
+
const command = new GetQueueAttributesCommand({
|
|
20009
|
+
QueueUrl: this.queueUrl,
|
|
20010
|
+
AttributeNames: ["QueueArn"]
|
|
20011
|
+
});
|
|
20012
|
+
await this.sqsClient.send(command);
|
|
20013
|
+
return true;
|
|
20014
|
+
} catch (error) {
|
|
20015
|
+
this.emit("connection_error", {
|
|
20016
|
+
replicator: this.name,
|
|
20017
|
+
error: error.message
|
|
20018
|
+
});
|
|
20019
|
+
return false;
|
|
20020
|
+
}
|
|
20021
|
+
}
|
|
20022
|
+
async getStatus() {
|
|
20023
|
+
const baseStatus = await super.getStatus();
|
|
20024
|
+
return {
|
|
20025
|
+
...baseStatus,
|
|
20026
|
+
connected: !!this.sqsClient,
|
|
20027
|
+
queueUrl: this.queueUrl,
|
|
20028
|
+
region: this.region,
|
|
20029
|
+
resources: this.resources,
|
|
20030
|
+
totalReplications: this.listenerCount("replicated"),
|
|
20031
|
+
totalErrors: this.listenerCount("replication_error")
|
|
20032
|
+
};
|
|
20033
|
+
}
|
|
20034
|
+
async cleanup() {
|
|
20035
|
+
if (this.sqsClient) {
|
|
20036
|
+
this.sqsClient.destroy();
|
|
20037
|
+
}
|
|
20038
|
+
await super.cleanup();
|
|
20039
|
+
}
|
|
20040
|
+
shouldReplicateResource(resourceName) {
|
|
20041
|
+
return this.resources.length === 0 || this.resources.includes(resourceName);
|
|
20042
|
+
}
|
|
20043
|
+
}
|
|
20044
|
+
|
|
20045
|
+
class BigqueryReplicator extends BaseReplicator {
|
|
20046
|
+
constructor(config = {}, resources = {}) {
|
|
20047
|
+
super(config);
|
|
20048
|
+
this.projectId = config.projectId;
|
|
20049
|
+
this.datasetId = config.datasetId;
|
|
20050
|
+
this.bigqueryClient = null;
|
|
20051
|
+
this.credentials = config.credentials;
|
|
20052
|
+
this.location = config.location || "US";
|
|
20053
|
+
this.logTable = config.logTable;
|
|
20054
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
20055
|
+
}
|
|
20056
|
+
parseResourcesConfig(resources) {
|
|
20057
|
+
const parsed = {};
|
|
20058
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
20059
|
+
if (typeof config === "string") {
|
|
20060
|
+
parsed[resourceName] = [{
|
|
20061
|
+
table: config,
|
|
20062
|
+
actions: ["insert"]
|
|
20063
|
+
}];
|
|
20064
|
+
} else if (Array.isArray(config)) {
|
|
20065
|
+
parsed[resourceName] = config.map((item) => {
|
|
20066
|
+
if (typeof item === "string") {
|
|
20067
|
+
return { table: item, actions: ["insert"] };
|
|
20068
|
+
}
|
|
20069
|
+
return {
|
|
20070
|
+
table: item.table,
|
|
20071
|
+
actions: item.actions || ["insert"]
|
|
20072
|
+
};
|
|
20073
|
+
});
|
|
20074
|
+
} else if (typeof config === "object") {
|
|
20075
|
+
parsed[resourceName] = [{
|
|
20076
|
+
table: config.table,
|
|
20077
|
+
actions: config.actions || ["insert"]
|
|
20078
|
+
}];
|
|
20079
|
+
}
|
|
20080
|
+
}
|
|
20081
|
+
return parsed;
|
|
20082
|
+
}
|
|
20083
|
+
validateConfig() {
|
|
20084
|
+
const errors = [];
|
|
20085
|
+
if (!this.projectId) errors.push("projectId is required");
|
|
20086
|
+
if (!this.datasetId) errors.push("datasetId is required");
|
|
20087
|
+
if (Object.keys(this.resources).length === 0) errors.push("At least one resource must be configured");
|
|
20088
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
20089
|
+
for (const tableConfig of tables) {
|
|
20090
|
+
if (!tableConfig.table) {
|
|
20091
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
20092
|
+
}
|
|
20093
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
20094
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
20095
|
+
}
|
|
20096
|
+
const validActions = ["insert", "update", "delete"];
|
|
20097
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
20098
|
+
if (invalidActions.length > 0) {
|
|
20099
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
20100
|
+
}
|
|
20101
|
+
}
|
|
20102
|
+
}
|
|
20103
|
+
return { isValid: errors.length === 0, errors };
|
|
20104
|
+
}
|
|
20105
|
+
async initialize(database) {
|
|
20106
|
+
await super.initialize(database);
|
|
20107
|
+
try {
|
|
20108
|
+
const { BigQuery } = await import('@google-cloud/bigquery');
|
|
20109
|
+
this.bigqueryClient = new BigQuery({
|
|
20110
|
+
projectId: this.projectId,
|
|
20111
|
+
credentials: this.credentials,
|
|
20112
|
+
location: this.location
|
|
20113
|
+
});
|
|
20114
|
+
this.emit("initialized", {
|
|
20115
|
+
replicator: this.name,
|
|
20116
|
+
projectId: this.projectId,
|
|
20117
|
+
datasetId: this.datasetId,
|
|
20118
|
+
resources: Object.keys(this.resources)
|
|
20119
|
+
});
|
|
20120
|
+
} catch (error) {
|
|
20121
|
+
this.emit("initialization_error", { replicator: this.name, error: error.message });
|
|
20122
|
+
throw error;
|
|
20123
|
+
}
|
|
20124
|
+
}
|
|
20125
|
+
shouldReplicateResource(resourceName) {
|
|
20126
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
20127
|
+
}
|
|
20128
|
+
shouldReplicateAction(resourceName, operation) {
|
|
20129
|
+
if (!this.resources[resourceName]) return false;
|
|
20130
|
+
return this.resources[resourceName].some(
|
|
20131
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
20132
|
+
);
|
|
20133
|
+
}
|
|
20134
|
+
getTablesForResource(resourceName, operation) {
|
|
20135
|
+
if (!this.resources[resourceName]) return [];
|
|
20136
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => tableConfig.table);
|
|
20137
|
+
}
|
|
20138
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
20139
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
20140
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
20141
|
+
}
|
|
20142
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
20143
|
+
return { skipped: true, reason: "action_not_included" };
|
|
20144
|
+
}
|
|
20145
|
+
const tables = this.getTablesForResource(resourceName, operation);
|
|
20146
|
+
if (tables.length === 0) {
|
|
20147
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
20148
|
+
}
|
|
20149
|
+
const results = [];
|
|
20150
|
+
const errors = [];
|
|
20151
|
+
try {
|
|
20152
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
20153
|
+
for (const tableId of tables) {
|
|
20154
|
+
try {
|
|
20155
|
+
const table = dataset.table(tableId);
|
|
20156
|
+
let job;
|
|
20157
|
+
if (operation === "insert") {
|
|
20158
|
+
const row = { ...data };
|
|
20159
|
+
job = await table.insert([row]);
|
|
20160
|
+
} else if (operation === "update") {
|
|
20161
|
+
const keys = Object.keys(data).filter((k) => k !== "id");
|
|
20162
|
+
const setClause = keys.map((k) => `${k}=@${k}`).join(", ");
|
|
20163
|
+
const params = { id };
|
|
20164
|
+
keys.forEach((k) => {
|
|
20165
|
+
params[k] = data[k];
|
|
20166
|
+
});
|
|
20167
|
+
const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableId}\` SET ${setClause} WHERE id=@id`;
|
|
20168
|
+
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
20169
|
+
query,
|
|
20170
|
+
params
|
|
20171
|
+
});
|
|
20172
|
+
await updateJob.getQueryResults();
|
|
20173
|
+
job = [updateJob];
|
|
20174
|
+
} else if (operation === "delete") {
|
|
20175
|
+
const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableId}\` WHERE id=@id`;
|
|
20176
|
+
const [deleteJob] = await this.bigqueryClient.createQueryJob({
|
|
20177
|
+
query,
|
|
20178
|
+
params: { id }
|
|
20179
|
+
});
|
|
20180
|
+
await deleteJob.getQueryResults();
|
|
20181
|
+
job = [deleteJob];
|
|
20182
|
+
} else {
|
|
20183
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
20184
|
+
}
|
|
20185
|
+
results.push({
|
|
20186
|
+
table: tableId,
|
|
20187
|
+
success: true,
|
|
20188
|
+
jobId: job[0]?.id
|
|
20189
|
+
});
|
|
20190
|
+
} catch (error) {
|
|
20191
|
+
errors.push({
|
|
20192
|
+
table: tableId,
|
|
20193
|
+
error: error.message
|
|
20194
|
+
});
|
|
20195
|
+
}
|
|
20196
|
+
}
|
|
20197
|
+
if (this.logTable) {
|
|
20198
|
+
try {
|
|
20199
|
+
const logTable = dataset.table(this.logTable);
|
|
20200
|
+
await logTable.insert([{
|
|
20201
|
+
resource_name: resourceName,
|
|
20202
|
+
operation,
|
|
20203
|
+
record_id: id,
|
|
20204
|
+
data: JSON.stringify(data),
|
|
20205
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20206
|
+
source: "s3db-replication"
|
|
20207
|
+
}]);
|
|
20208
|
+
} catch (error) {
|
|
20209
|
+
console.warn(`Failed to log operation to ${this.logTable}:`, error.message);
|
|
20210
|
+
}
|
|
20211
|
+
}
|
|
20212
|
+
const success = errors.length === 0;
|
|
20213
|
+
this.emit("replicated", {
|
|
20214
|
+
replicator: this.name,
|
|
20215
|
+
resourceName,
|
|
20216
|
+
operation,
|
|
20217
|
+
id,
|
|
20218
|
+
tables,
|
|
20219
|
+
results,
|
|
20220
|
+
errors,
|
|
20221
|
+
success
|
|
20222
|
+
});
|
|
20223
|
+
return {
|
|
20224
|
+
success,
|
|
20225
|
+
results,
|
|
20226
|
+
errors,
|
|
20227
|
+
tables
|
|
20228
|
+
};
|
|
20229
|
+
} catch (error) {
|
|
20230
|
+
this.emit("replication_error", {
|
|
20231
|
+
replicator: this.name,
|
|
20232
|
+
resourceName,
|
|
20233
|
+
operation,
|
|
20234
|
+
id,
|
|
20235
|
+
error: error.message
|
|
20236
|
+
});
|
|
20237
|
+
return { success: false, error: error.message };
|
|
20238
|
+
}
|
|
20239
|
+
}
|
|
20240
|
+
async replicateBatch(resourceName, records) {
|
|
20241
|
+
const results = [];
|
|
20242
|
+
const errors = [];
|
|
20243
|
+
for (const record of records) {
|
|
20244
|
+
try {
|
|
20245
|
+
const res = await this.replicate(
|
|
20246
|
+
resourceName,
|
|
20247
|
+
record.operation,
|
|
20248
|
+
record.data,
|
|
20249
|
+
record.id,
|
|
20250
|
+
record.beforeData
|
|
20251
|
+
);
|
|
20252
|
+
results.push(res);
|
|
20253
|
+
} catch (err) {
|
|
20254
|
+
errors.push({ id: record.id, error: err.message });
|
|
20255
|
+
}
|
|
20256
|
+
}
|
|
20257
|
+
return {
|
|
20258
|
+
success: errors.length === 0,
|
|
20259
|
+
results,
|
|
20260
|
+
errors
|
|
20261
|
+
};
|
|
20262
|
+
}
|
|
20263
|
+
async testConnection() {
|
|
20264
|
+
try {
|
|
20265
|
+
if (!this.bigqueryClient) await this.initialize();
|
|
20266
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
20267
|
+
await dataset.getMetadata();
|
|
20268
|
+
return true;
|
|
20269
|
+
} catch (error) {
|
|
20270
|
+
this.emit("connection_error", { replicator: this.name, error: error.message });
|
|
20271
|
+
return false;
|
|
20272
|
+
}
|
|
20273
|
+
}
|
|
20274
|
+
async cleanup() {
|
|
20275
|
+
}
|
|
20276
|
+
getStatus() {
|
|
20277
|
+
return {
|
|
20278
|
+
...super.getStatus(),
|
|
20279
|
+
projectId: this.projectId,
|
|
20280
|
+
datasetId: this.datasetId,
|
|
20281
|
+
resources: this.resources,
|
|
20282
|
+
logTable: this.logTable
|
|
20283
|
+
};
|
|
20284
|
+
}
|
|
20285
|
+
}
|
|
20286
|
+
|
|
20287
|
+
class PostgresReplicator extends BaseReplicator {
|
|
20288
|
+
constructor(config = {}, resources = {}) {
|
|
20289
|
+
super(config);
|
|
20290
|
+
this.connectionString = config.connectionString;
|
|
20291
|
+
this.host = config.host;
|
|
20292
|
+
this.port = config.port || 5432;
|
|
20293
|
+
this.database = config.database;
|
|
20294
|
+
this.user = config.user;
|
|
20295
|
+
this.password = config.password;
|
|
20296
|
+
this.client = null;
|
|
20297
|
+
this.ssl = config.ssl;
|
|
20298
|
+
this.logTable = config.logTable;
|
|
20299
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
20300
|
+
}
|
|
20301
|
+
parseResourcesConfig(resources) {
|
|
20302
|
+
const parsed = {};
|
|
20303
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
20304
|
+
if (typeof config === "string") {
|
|
20305
|
+
parsed[resourceName] = [{
|
|
20306
|
+
table: config,
|
|
20307
|
+
actions: ["insert"]
|
|
20308
|
+
}];
|
|
20309
|
+
} else if (Array.isArray(config)) {
|
|
20310
|
+
parsed[resourceName] = config.map((item) => {
|
|
20311
|
+
if (typeof item === "string") {
|
|
20312
|
+
return { table: item, actions: ["insert"] };
|
|
20313
|
+
}
|
|
20314
|
+
return {
|
|
20315
|
+
table: item.table,
|
|
20316
|
+
actions: item.actions || ["insert"]
|
|
20317
|
+
};
|
|
20318
|
+
});
|
|
20319
|
+
} else if (typeof config === "object") {
|
|
20320
|
+
parsed[resourceName] = [{
|
|
20321
|
+
table: config.table,
|
|
20322
|
+
actions: config.actions || ["insert"]
|
|
20323
|
+
}];
|
|
20324
|
+
}
|
|
20325
|
+
}
|
|
20326
|
+
return parsed;
|
|
20327
|
+
}
|
|
20328
|
+
validateConfig() {
|
|
20329
|
+
const errors = [];
|
|
20330
|
+
if (!this.connectionString && (!this.host || !this.database)) {
|
|
20331
|
+
errors.push("Either connectionString or host+database must be provided");
|
|
20332
|
+
}
|
|
20333
|
+
if (Object.keys(this.resources).length === 0) {
|
|
20334
|
+
errors.push("At least one resource must be configured");
|
|
20335
|
+
}
|
|
20336
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
20337
|
+
for (const tableConfig of tables) {
|
|
20338
|
+
if (!tableConfig.table) {
|
|
20339
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
20340
|
+
}
|
|
20341
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
20342
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
20343
|
+
}
|
|
20344
|
+
const validActions = ["insert", "update", "delete"];
|
|
20345
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
20346
|
+
if (invalidActions.length > 0) {
|
|
20347
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
20348
|
+
}
|
|
20349
|
+
}
|
|
20350
|
+
}
|
|
20351
|
+
return { isValid: errors.length === 0, errors };
|
|
20352
|
+
}
|
|
20353
|
+
async initialize(database) {
|
|
20354
|
+
await super.initialize(database);
|
|
20355
|
+
try {
|
|
20356
|
+
const { Client } = await import('pg');
|
|
20357
|
+
const config = this.connectionString ? {
|
|
20358
|
+
connectionString: this.connectionString,
|
|
20359
|
+
ssl: this.ssl
|
|
20360
|
+
} : {
|
|
20361
|
+
host: this.host,
|
|
20362
|
+
port: this.port,
|
|
20363
|
+
database: this.database,
|
|
20364
|
+
user: this.user,
|
|
20365
|
+
password: this.password,
|
|
20366
|
+
ssl: this.ssl
|
|
20367
|
+
};
|
|
20368
|
+
this.client = new Client(config);
|
|
20369
|
+
await this.client.connect();
|
|
20370
|
+
if (this.logTable) {
|
|
20371
|
+
await this.createLogTableIfNotExists();
|
|
20372
|
+
}
|
|
20373
|
+
this.emit("initialized", {
|
|
20374
|
+
replicator: this.name,
|
|
20375
|
+
database: this.database || "postgres",
|
|
20376
|
+
resources: Object.keys(this.resources)
|
|
20377
|
+
});
|
|
20378
|
+
} catch (error) {
|
|
20379
|
+
this.emit("initialization_error", {
|
|
20380
|
+
replicator: this.name,
|
|
20381
|
+
error: error.message
|
|
20382
|
+
});
|
|
20383
|
+
throw error;
|
|
20384
|
+
}
|
|
20385
|
+
}
|
|
20386
|
+
async createLogTableIfNotExists() {
|
|
20387
|
+
const createTableQuery = `
|
|
20388
|
+
CREATE TABLE IF NOT EXISTS ${this.logTable} (
|
|
20389
|
+
id SERIAL PRIMARY KEY,
|
|
20390
|
+
resource_name VARCHAR(255) NOT NULL,
|
|
20391
|
+
operation VARCHAR(50) NOT NULL,
|
|
20392
|
+
record_id VARCHAR(255) NOT NULL,
|
|
20393
|
+
data JSONB,
|
|
20394
|
+
timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
20395
|
+
source VARCHAR(100) DEFAULT 's3db-replication',
|
|
20396
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
20397
|
+
);
|
|
20398
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_resource_name ON ${this.logTable}(resource_name);
|
|
20399
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_operation ON ${this.logTable}(operation);
|
|
20400
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_record_id ON ${this.logTable}(record_id);
|
|
20401
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_timestamp ON ${this.logTable}(timestamp);
|
|
20402
|
+
`;
|
|
20403
|
+
await this.client.query(createTableQuery);
|
|
20404
|
+
}
|
|
20405
|
+
shouldReplicateResource(resourceName) {
|
|
20406
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
20407
|
+
}
|
|
20408
|
+
shouldReplicateAction(resourceName, operation) {
|
|
20409
|
+
if (!this.resources[resourceName]) return false;
|
|
20410
|
+
return this.resources[resourceName].some(
|
|
20411
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
20412
|
+
);
|
|
20413
|
+
}
|
|
20414
|
+
getTablesForResource(resourceName, operation) {
|
|
20415
|
+
if (!this.resources[resourceName]) return [];
|
|
20416
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => tableConfig.table);
|
|
20417
|
+
}
|
|
20418
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
20419
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
20420
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
20421
|
+
}
|
|
20422
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
20423
|
+
return { skipped: true, reason: "action_not_included" };
|
|
20424
|
+
}
|
|
20425
|
+
const tables = this.getTablesForResource(resourceName, operation);
|
|
20426
|
+
if (tables.length === 0) {
|
|
20427
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
20428
|
+
}
|
|
20429
|
+
const results = [];
|
|
20430
|
+
const errors = [];
|
|
20431
|
+
try {
|
|
20432
|
+
for (const table of tables) {
|
|
20433
|
+
try {
|
|
20434
|
+
let result;
|
|
20435
|
+
if (operation === "insert") {
|
|
20436
|
+
const keys = Object.keys(data);
|
|
20437
|
+
const values = keys.map((k) => data[k]);
|
|
20438
|
+
const columns = keys.map((k) => `"${k}"`).join(", ");
|
|
20439
|
+
const params = keys.map((_, i) => `$${i + 1}`).join(", ");
|
|
20440
|
+
const sql = `INSERT INTO ${table} (${columns}) VALUES (${params}) ON CONFLICT (id) DO NOTHING RETURNING *`;
|
|
20441
|
+
result = await this.client.query(sql, values);
|
|
20442
|
+
} else if (operation === "update") {
|
|
20443
|
+
const keys = Object.keys(data).filter((k) => k !== "id");
|
|
20444
|
+
const setClause = keys.map((k, i) => `"${k}"=$${i + 1}`).join(", ");
|
|
20445
|
+
const values = keys.map((k) => data[k]);
|
|
20446
|
+
values.push(id);
|
|
20447
|
+
const sql = `UPDATE ${table} SET ${setClause} WHERE id=$${keys.length + 1} RETURNING *`;
|
|
20448
|
+
result = await this.client.query(sql, values);
|
|
20449
|
+
} else if (operation === "delete") {
|
|
20450
|
+
const sql = `DELETE FROM ${table} WHERE id=$1 RETURNING *`;
|
|
20451
|
+
result = await this.client.query(sql, [id]);
|
|
20452
|
+
} else {
|
|
20453
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
20454
|
+
}
|
|
20455
|
+
results.push({
|
|
20456
|
+
table,
|
|
20457
|
+
success: true,
|
|
20458
|
+
rows: result.rows,
|
|
20459
|
+
rowCount: result.rowCount
|
|
20460
|
+
});
|
|
20461
|
+
} catch (error) {
|
|
20462
|
+
errors.push({
|
|
20463
|
+
table,
|
|
20464
|
+
error: error.message
|
|
20465
|
+
});
|
|
20466
|
+
}
|
|
20467
|
+
}
|
|
20468
|
+
if (this.logTable) {
|
|
20469
|
+
try {
|
|
20470
|
+
await this.client.query(
|
|
20471
|
+
`INSERT INTO ${this.logTable} (resource_name, operation, record_id, data, timestamp, source) VALUES ($1, $2, $3, $4, $5, $6)`,
|
|
20472
|
+
[resourceName, operation, id, JSON.stringify(data), (/* @__PURE__ */ new Date()).toISOString(), "s3db-replication"]
|
|
20473
|
+
);
|
|
20474
|
+
} catch (error) {
|
|
20475
|
+
console.warn(`Failed to log operation to ${this.logTable}:`, error.message);
|
|
20476
|
+
}
|
|
20477
|
+
}
|
|
20478
|
+
const success = errors.length === 0;
|
|
20479
|
+
this.emit("replicated", {
|
|
20480
|
+
replicator: this.name,
|
|
20481
|
+
resourceName,
|
|
20482
|
+
operation,
|
|
20483
|
+
id,
|
|
20484
|
+
tables,
|
|
20485
|
+
results,
|
|
20486
|
+
errors,
|
|
20487
|
+
success
|
|
20488
|
+
});
|
|
20489
|
+
return {
|
|
20490
|
+
success,
|
|
20491
|
+
results,
|
|
20492
|
+
errors,
|
|
20493
|
+
tables
|
|
20494
|
+
};
|
|
20495
|
+
} catch (error) {
|
|
20496
|
+
this.emit("replication_error", {
|
|
20497
|
+
replicator: this.name,
|
|
20498
|
+
resourceName,
|
|
20499
|
+
operation,
|
|
20500
|
+
id,
|
|
20501
|
+
error: error.message
|
|
20502
|
+
});
|
|
20503
|
+
return { success: false, error: error.message };
|
|
20504
|
+
}
|
|
20505
|
+
}
|
|
20506
|
+
async replicateBatch(resourceName, records) {
|
|
20507
|
+
const results = [];
|
|
20508
|
+
const errors = [];
|
|
20509
|
+
for (const record of records) {
|
|
17911
20510
|
try {
|
|
17912
|
-
const
|
|
17913
|
-
|
|
20511
|
+
const res = await this.replicate(
|
|
20512
|
+
resourceName,
|
|
20513
|
+
record.operation,
|
|
20514
|
+
record.data,
|
|
20515
|
+
record.id,
|
|
20516
|
+
record.beforeData
|
|
20517
|
+
);
|
|
20518
|
+
results.push(res);
|
|
17914
20519
|
} catch (err) {
|
|
17915
|
-
|
|
20520
|
+
errors.push({ id: record.id, error: err.message });
|
|
17916
20521
|
}
|
|
17917
|
-
|
|
17918
|
-
|
|
17919
|
-
|
|
20522
|
+
}
|
|
20523
|
+
return {
|
|
20524
|
+
success: errors.length === 0,
|
|
20525
|
+
results,
|
|
20526
|
+
errors
|
|
17920
20527
|
};
|
|
17921
|
-
|
|
17922
|
-
|
|
17923
|
-
|
|
17924
|
-
|
|
17925
|
-
|
|
17926
|
-
|
|
17927
|
-
|
|
17928
|
-
|
|
20528
|
+
}
|
|
20529
|
+
async testConnection() {
|
|
20530
|
+
try {
|
|
20531
|
+
if (!this.client) await this.initialize();
|
|
20532
|
+
await this.client.query("SELECT 1");
|
|
20533
|
+
return true;
|
|
20534
|
+
} catch (error) {
|
|
20535
|
+
this.emit("connection_error", { replicator: this.name, error: error.message });
|
|
20536
|
+
return false;
|
|
20537
|
+
}
|
|
20538
|
+
}
|
|
20539
|
+
async cleanup() {
|
|
20540
|
+
if (this.client) await this.client.end();
|
|
20541
|
+
}
|
|
20542
|
+
getStatus() {
|
|
20543
|
+
return {
|
|
20544
|
+
...super.getStatus(),
|
|
20545
|
+
database: this.database || "postgres",
|
|
20546
|
+
resources: this.resources,
|
|
20547
|
+
logTable: this.logTable
|
|
17929
20548
|
};
|
|
17930
|
-
|
|
17931
|
-
|
|
17932
|
-
|
|
17933
|
-
|
|
20549
|
+
}
|
|
20550
|
+
}
|
|
20551
|
+
|
|
20552
|
+
const REPLICATOR_DRIVERS = {
|
|
20553
|
+
s3db: S3dbReplicator,
|
|
20554
|
+
sqs: SqsReplicator,
|
|
20555
|
+
bigquery: BigqueryReplicator,
|
|
20556
|
+
postgres: PostgresReplicator
|
|
20557
|
+
};
|
|
20558
|
+
function createReplicator(driver, config = {}, resources = []) {
|
|
20559
|
+
const ReplicatorClass = REPLICATOR_DRIVERS[driver];
|
|
20560
|
+
if (!ReplicatorClass) {
|
|
20561
|
+
throw new Error(`Unknown replicator driver: ${driver}. Available drivers: ${Object.keys(REPLICATOR_DRIVERS).join(", ")}`);
|
|
20562
|
+
}
|
|
20563
|
+
return new ReplicatorClass(config, resources);
|
|
20564
|
+
}
|
|
20565
|
+
function validateReplicatorConfig(driver, config, resources = []) {
|
|
20566
|
+
const replicator = createReplicator(driver, config, resources);
|
|
20567
|
+
return replicator.validateConfig();
|
|
20568
|
+
}
|
|
20569
|
+
|
|
20570
|
+
const gzipAsync = promisify(gzip);
|
|
20571
|
+
const gunzipAsync = promisify(gunzip);
|
|
20572
|
+
class ReplicationPlugin extends Plugin {
|
|
20573
|
+
constructor(options = {}) {
|
|
20574
|
+
super();
|
|
20575
|
+
this.config = {
|
|
20576
|
+
enabled: options.enabled !== false,
|
|
20577
|
+
replicators: options.replicators || [],
|
|
20578
|
+
syncMode: options.syncMode || "async",
|
|
20579
|
+
// 'sync' or 'async'
|
|
20580
|
+
retryAttempts: options.retryAttempts || 3,
|
|
20581
|
+
retryDelay: options.retryDelay || 1e3,
|
|
20582
|
+
// ms
|
|
20583
|
+
batchSize: options.batchSize || 10,
|
|
20584
|
+
compression: options.compression || false,
|
|
20585
|
+
// Enable compression
|
|
20586
|
+
compressionLevel: options.compressionLevel || 6,
|
|
20587
|
+
// 0-9
|
|
20588
|
+
...options
|
|
17934
20589
|
};
|
|
17935
|
-
|
|
17936
|
-
|
|
17937
|
-
|
|
20590
|
+
this.replicators = [];
|
|
20591
|
+
this.queue = [];
|
|
20592
|
+
this.isProcessing = false;
|
|
20593
|
+
this.stats = {
|
|
20594
|
+
totalOperations: 0,
|
|
20595
|
+
successfulOperations: 0,
|
|
20596
|
+
failedOperations: 0,
|
|
20597
|
+
lastSync: null
|
|
20598
|
+
};
|
|
20599
|
+
}
|
|
20600
|
+
/**
|
|
20601
|
+
* Process data according to replication mode
|
|
20602
|
+
*/
|
|
20603
|
+
processDataForReplication(data, metadata = {}) {
|
|
20604
|
+
switch (this.config.replicationMode) {
|
|
20605
|
+
case "exact-copy":
|
|
20606
|
+
return {
|
|
20607
|
+
body: data,
|
|
20608
|
+
metadata
|
|
20609
|
+
};
|
|
20610
|
+
case "just-metadata":
|
|
20611
|
+
return {
|
|
20612
|
+
body: null,
|
|
20613
|
+
metadata
|
|
20614
|
+
};
|
|
20615
|
+
case "all-in-body":
|
|
20616
|
+
return {
|
|
20617
|
+
body: {
|
|
20618
|
+
data,
|
|
20619
|
+
metadata,
|
|
20620
|
+
replicationMode: this.config.replicationMode,
|
|
20621
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
20622
|
+
},
|
|
20623
|
+
metadata: {
|
|
20624
|
+
replicationMode: this.config.replicationMode,
|
|
20625
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
20626
|
+
}
|
|
20627
|
+
};
|
|
20628
|
+
default:
|
|
20629
|
+
return {
|
|
20630
|
+
body: data,
|
|
20631
|
+
metadata
|
|
20632
|
+
};
|
|
20633
|
+
}
|
|
20634
|
+
}
|
|
20635
|
+
/**
|
|
20636
|
+
* Compress data if compression is enabled
|
|
20637
|
+
*/
|
|
20638
|
+
async compressData(data) {
|
|
20639
|
+
if (!this.config.compression || !data) {
|
|
20640
|
+
return data;
|
|
20641
|
+
}
|
|
20642
|
+
try {
|
|
20643
|
+
const jsonString = JSON.stringify(data);
|
|
20644
|
+
const compressed = await gzipAsync(jsonString, { level: this.config.compressionLevel });
|
|
20645
|
+
return compressed.toString("base64");
|
|
20646
|
+
} catch (error) {
|
|
20647
|
+
this.emit("replication.compression.failed", { error, data });
|
|
20648
|
+
return data;
|
|
20649
|
+
}
|
|
20650
|
+
}
|
|
20651
|
+
/**
|
|
20652
|
+
* Decompress data if it was compressed
|
|
20653
|
+
*/
|
|
20654
|
+
async decompressData(data) {
|
|
20655
|
+
if (!this.config.compression || !data) {
|
|
20656
|
+
return data;
|
|
20657
|
+
}
|
|
20658
|
+
try {
|
|
20659
|
+
if (typeof data === "string" && data.startsWith("H4sI")) {
|
|
20660
|
+
const buffer = Buffer.from(data, "base64");
|
|
20661
|
+
const decompressed = await gunzipAsync(buffer);
|
|
20662
|
+
return JSON.parse(decompressed.toString());
|
|
20663
|
+
}
|
|
20664
|
+
return data;
|
|
20665
|
+
} catch (error) {
|
|
20666
|
+
this.emit("replication.decompression.failed", { error, data });
|
|
17938
20667
|
return data;
|
|
20668
|
+
}
|
|
20669
|
+
}
|
|
20670
|
+
async setup(database) {
|
|
20671
|
+
this.database = database;
|
|
20672
|
+
if (!this.config.enabled) {
|
|
20673
|
+
return;
|
|
20674
|
+
}
|
|
20675
|
+
if (this.config.replicators && this.config.replicators.length > 0) {
|
|
20676
|
+
await this.initializeReplicators();
|
|
20677
|
+
}
|
|
20678
|
+
if (!database.resources.replication_logs) {
|
|
20679
|
+
this.replicationLog = await database.createResource({
|
|
20680
|
+
name: "replication_logs",
|
|
20681
|
+
attributes: {
|
|
20682
|
+
id: "string|required",
|
|
20683
|
+
resourceName: "string|required",
|
|
20684
|
+
operation: "string|required",
|
|
20685
|
+
recordId: "string|required",
|
|
20686
|
+
replicatorId: "string|required",
|
|
20687
|
+
status: "string|required",
|
|
20688
|
+
attempts: "number|required",
|
|
20689
|
+
lastAttempt: "string|required",
|
|
20690
|
+
error: "string|required",
|
|
20691
|
+
data: "object|required",
|
|
20692
|
+
timestamp: "string|required"
|
|
20693
|
+
}
|
|
20694
|
+
});
|
|
20695
|
+
} else {
|
|
20696
|
+
this.replicationLog = database.resources.replication_logs;
|
|
20697
|
+
}
|
|
20698
|
+
for (const resourceName in database.resources) {
|
|
20699
|
+
if (resourceName !== "replication_logs") {
|
|
20700
|
+
this.installHooks(database.resources[resourceName]);
|
|
20701
|
+
}
|
|
20702
|
+
}
|
|
20703
|
+
const originalCreateResource = database.createResource.bind(database);
|
|
20704
|
+
database.createResource = async (config) => {
|
|
20705
|
+
const resource = await originalCreateResource(config);
|
|
20706
|
+
if (resource && resource.name !== "replication_logs") {
|
|
20707
|
+
this.installHooks(resource);
|
|
20708
|
+
}
|
|
20709
|
+
return resource;
|
|
17939
20710
|
};
|
|
17940
|
-
|
|
17941
|
-
|
|
17942
|
-
|
|
20711
|
+
this.startQueueProcessor();
|
|
20712
|
+
}
|
|
20713
|
+
async initializeReplicators() {
|
|
20714
|
+
for (const replicatorConfig of this.config.replicators) {
|
|
20715
|
+
try {
|
|
20716
|
+
const { driver, config: replicatorConfigData, resources = [] } = replicatorConfig;
|
|
20717
|
+
const validation = validateReplicatorConfig(driver, replicatorConfigData, resources);
|
|
20718
|
+
if (!validation.isValid) {
|
|
20719
|
+
this.emit("replicator.validation.failed", {
|
|
20720
|
+
driver,
|
|
20721
|
+
errors: validation.errors
|
|
20722
|
+
});
|
|
20723
|
+
continue;
|
|
20724
|
+
}
|
|
20725
|
+
const replicator = createReplicator(driver, replicatorConfigData, resources);
|
|
20726
|
+
await replicator.initialize(this.database);
|
|
20727
|
+
replicator.on("replicated", (data) => {
|
|
20728
|
+
this.emit("replication.success", data);
|
|
20729
|
+
});
|
|
20730
|
+
replicator.on("replication_error", (data) => {
|
|
20731
|
+
this.emit("replication.failed", data);
|
|
20732
|
+
});
|
|
20733
|
+
this.replicators.push({
|
|
20734
|
+
id: `${driver}-${Date.now()}`,
|
|
20735
|
+
driver,
|
|
20736
|
+
config: replicatorConfigData,
|
|
20737
|
+
resources,
|
|
20738
|
+
instance: replicator
|
|
20739
|
+
});
|
|
20740
|
+
this.emit("replicator.initialized", {
|
|
20741
|
+
driver,
|
|
20742
|
+
config: replicatorConfigData,
|
|
20743
|
+
resources
|
|
20744
|
+
});
|
|
20745
|
+
} catch (error) {
|
|
20746
|
+
this.emit("replicator.initialization.failed", {
|
|
20747
|
+
driver: replicatorConfig.driver,
|
|
20748
|
+
error: error.message
|
|
20749
|
+
});
|
|
20750
|
+
}
|
|
20751
|
+
}
|
|
20752
|
+
}
|
|
20753
|
+
async start() {
|
|
20754
|
+
}
|
|
20755
|
+
async stop() {
|
|
20756
|
+
this.isProcessing = false;
|
|
20757
|
+
await this.processQueue();
|
|
20758
|
+
}
|
|
20759
|
+
installHooks(resource) {
|
|
20760
|
+
if (!resource || resource.name === "replication_logs") return;
|
|
20761
|
+
const originalDataMap = /* @__PURE__ */ new Map();
|
|
20762
|
+
resource.addHook("afterInsert", async (data) => {
|
|
20763
|
+
await this.queueReplication(resource.name, "insert", data.id, data);
|
|
20764
|
+
return data;
|
|
20765
|
+
});
|
|
20766
|
+
resource.addHook("preUpdate", async (data) => {
|
|
20767
|
+
if (data.id) {
|
|
20768
|
+
try {
|
|
20769
|
+
const originalData = await resource.get(data.id);
|
|
20770
|
+
originalDataMap.set(data.id, originalData);
|
|
20771
|
+
} catch (error) {
|
|
20772
|
+
originalDataMap.set(data.id, { id: data.id });
|
|
20773
|
+
}
|
|
20774
|
+
}
|
|
20775
|
+
return data;
|
|
20776
|
+
});
|
|
20777
|
+
resource.addHook("afterUpdate", async (data) => {
|
|
20778
|
+
const beforeData = originalDataMap.get(data.id);
|
|
20779
|
+
await this.queueReplication(resource.name, "update", data.id, data, beforeData);
|
|
20780
|
+
originalDataMap.delete(data.id);
|
|
17943
20781
|
return data;
|
|
20782
|
+
});
|
|
20783
|
+
resource.addHook("afterDelete", async (data) => {
|
|
20784
|
+
await this.queueReplication(resource.name, "delete", data.id, data);
|
|
20785
|
+
return data;
|
|
20786
|
+
});
|
|
20787
|
+
const originalDeleteMany = resource.deleteMany.bind(resource);
|
|
20788
|
+
resource.deleteMany = async (ids) => {
|
|
20789
|
+
const result = await originalDeleteMany(ids);
|
|
20790
|
+
if (result && result.length > 0) {
|
|
20791
|
+
for (const id of ids) {
|
|
20792
|
+
await this.queueReplication(resource.name, "delete", id, { id });
|
|
20793
|
+
}
|
|
20794
|
+
}
|
|
20795
|
+
return result;
|
|
20796
|
+
};
|
|
20797
|
+
}
|
|
20798
|
+
async queueReplication(resourceName, operation, recordId, data, beforeData = null) {
|
|
20799
|
+
if (!this.config.enabled) {
|
|
20800
|
+
return;
|
|
20801
|
+
}
|
|
20802
|
+
if (this.replicators.length === 0) {
|
|
20803
|
+
return;
|
|
20804
|
+
}
|
|
20805
|
+
const applicableReplicators = this.replicators.filter(
|
|
20806
|
+
(replicator) => replicator.instance.shouldReplicateResource(resourceName)
|
|
20807
|
+
);
|
|
20808
|
+
if (applicableReplicators.length === 0) {
|
|
20809
|
+
return;
|
|
20810
|
+
}
|
|
20811
|
+
const item = {
|
|
20812
|
+
id: `repl-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
20813
|
+
resourceName,
|
|
20814
|
+
operation,
|
|
20815
|
+
recordId,
|
|
20816
|
+
data: lodashEs.isPlainObject(data) ? data : { raw: data },
|
|
20817
|
+
beforeData: beforeData ? lodashEs.isPlainObject(beforeData) ? beforeData : { raw: beforeData } : null,
|
|
20818
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20819
|
+
attempts: 0
|
|
20820
|
+
};
|
|
20821
|
+
const logId = await this.logReplication(item);
|
|
20822
|
+
if (this.config.syncMode === "sync") {
|
|
20823
|
+
try {
|
|
20824
|
+
const result = await this.processReplicationItem(item);
|
|
20825
|
+
if (logId) {
|
|
20826
|
+
await this.updateReplicationLog(logId, {
|
|
20827
|
+
status: result.success ? "success" : "failed",
|
|
20828
|
+
attempts: 1,
|
|
20829
|
+
error: result.success ? "" : JSON.stringify(result.results)
|
|
20830
|
+
});
|
|
20831
|
+
}
|
|
20832
|
+
this.stats.totalOperations++;
|
|
20833
|
+
if (result.success) {
|
|
20834
|
+
this.stats.successfulOperations++;
|
|
20835
|
+
} else {
|
|
20836
|
+
this.stats.failedOperations++;
|
|
20837
|
+
}
|
|
20838
|
+
} catch (error) {
|
|
20839
|
+
if (logId) {
|
|
20840
|
+
await this.updateReplicationLog(logId, {
|
|
20841
|
+
status: "failed",
|
|
20842
|
+
attempts: 1,
|
|
20843
|
+
error: error.message
|
|
20844
|
+
});
|
|
20845
|
+
}
|
|
20846
|
+
this.stats.failedOperations++;
|
|
20847
|
+
}
|
|
20848
|
+
} else {
|
|
20849
|
+
this.queue.push(item);
|
|
20850
|
+
this.emit("replication.queued", { item, queueLength: this.queue.length });
|
|
20851
|
+
}
|
|
20852
|
+
}
|
|
20853
|
+
async processReplicationItem(item) {
|
|
20854
|
+
const { resourceName, operation, recordId, data, beforeData } = item;
|
|
20855
|
+
const applicableReplicators = this.replicators.filter(
|
|
20856
|
+
(replicator) => replicator.instance.shouldReplicateResource(resourceName)
|
|
20857
|
+
);
|
|
20858
|
+
if (applicableReplicators.length === 0) {
|
|
20859
|
+
return { success: true, skipped: true, reason: "no_applicable_replicators" };
|
|
20860
|
+
}
|
|
20861
|
+
const results = [];
|
|
20862
|
+
for (const replicator of applicableReplicators) {
|
|
20863
|
+
try {
|
|
20864
|
+
const result = await replicator.instance.replicate(resourceName, operation, data, recordId, beforeData);
|
|
20865
|
+
results.push({
|
|
20866
|
+
replicatorId: replicator.id,
|
|
20867
|
+
driver: replicator.driver,
|
|
20868
|
+
success: result.success,
|
|
20869
|
+
error: result.error,
|
|
20870
|
+
skipped: result.skipped
|
|
20871
|
+
});
|
|
20872
|
+
} catch (error) {
|
|
20873
|
+
results.push({
|
|
20874
|
+
replicatorId: replicator.id,
|
|
20875
|
+
driver: replicator.driver,
|
|
20876
|
+
success: false,
|
|
20877
|
+
error: error.message
|
|
20878
|
+
});
|
|
20879
|
+
}
|
|
20880
|
+
}
|
|
20881
|
+
return {
|
|
20882
|
+
success: results.every((r) => r.success || r.skipped),
|
|
20883
|
+
results
|
|
20884
|
+
};
|
|
20885
|
+
}
|
|
20886
|
+
async logReplication(item) {
|
|
20887
|
+
if (!this.replicationLog) return;
|
|
20888
|
+
try {
|
|
20889
|
+
const logId = `log-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
20890
|
+
await this.replicationLog.insert({
|
|
20891
|
+
id: logId,
|
|
20892
|
+
resourceName: item.resourceName,
|
|
20893
|
+
operation: item.operation,
|
|
20894
|
+
recordId: item.recordId,
|
|
20895
|
+
replicatorId: "all",
|
|
20896
|
+
// Will be updated with specific replicator results
|
|
20897
|
+
status: "queued",
|
|
20898
|
+
attempts: 0,
|
|
20899
|
+
lastAttempt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20900
|
+
error: "",
|
|
20901
|
+
data: lodashEs.isPlainObject(item.data) ? item.data : { raw: item.data },
|
|
20902
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
20903
|
+
});
|
|
20904
|
+
return logId;
|
|
20905
|
+
} catch (error) {
|
|
20906
|
+
this.emit("replication.log.failed", { error: error.message, item });
|
|
20907
|
+
return null;
|
|
20908
|
+
}
|
|
20909
|
+
}
|
|
20910
|
+
async updateReplicationLog(logId, updates) {
|
|
20911
|
+
if (!this.replicationLog) return;
|
|
20912
|
+
try {
|
|
20913
|
+
await this.replicationLog.update(logId, {
|
|
20914
|
+
...updates,
|
|
20915
|
+
lastAttempt: (/* @__PURE__ */ new Date()).toISOString()
|
|
20916
|
+
});
|
|
20917
|
+
} catch (error) {
|
|
20918
|
+
this.emit("replication.updateLog.failed", { error: error.message, logId, updates });
|
|
20919
|
+
}
|
|
20920
|
+
}
|
|
20921
|
+
startQueueProcessor() {
|
|
20922
|
+
if (this.isProcessing) return;
|
|
20923
|
+
this.isProcessing = true;
|
|
20924
|
+
this.processQueueLoop();
|
|
20925
|
+
}
|
|
20926
|
+
async processQueueLoop() {
|
|
20927
|
+
while (this.isProcessing) {
|
|
20928
|
+
if (this.queue.length > 0) {
|
|
20929
|
+
const batch = this.queue.splice(0, this.config.batchSize);
|
|
20930
|
+
for (const item of batch) {
|
|
20931
|
+
await this.processReplicationItem(item);
|
|
20932
|
+
}
|
|
20933
|
+
} else {
|
|
20934
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
20935
|
+
}
|
|
20936
|
+
}
|
|
20937
|
+
}
|
|
20938
|
+
async processQueue() {
|
|
20939
|
+
if (this.queue.length === 0) return;
|
|
20940
|
+
const item = this.queue.shift();
|
|
20941
|
+
let attempts = 0;
|
|
20942
|
+
let lastError = null;
|
|
20943
|
+
while (attempts < this.config.retryAttempts) {
|
|
20944
|
+
try {
|
|
20945
|
+
attempts++;
|
|
20946
|
+
this.emit("replication.retry.started", {
|
|
20947
|
+
item,
|
|
20948
|
+
attempt: attempts,
|
|
20949
|
+
maxAttempts: this.config.retryAttempts
|
|
20950
|
+
});
|
|
20951
|
+
const result = await this.processReplicationItem(item);
|
|
20952
|
+
if (result.success) {
|
|
20953
|
+
this.stats.successfulOperations++;
|
|
20954
|
+
this.emit("replication.success", {
|
|
20955
|
+
item,
|
|
20956
|
+
attempts,
|
|
20957
|
+
results: result.results,
|
|
20958
|
+
stats: this.stats
|
|
20959
|
+
});
|
|
20960
|
+
return;
|
|
20961
|
+
} else {
|
|
20962
|
+
lastError = result.results;
|
|
20963
|
+
if (attempts < this.config.retryAttempts) {
|
|
20964
|
+
await new Promise((resolve) => setTimeout(resolve, this.config.retryDelay * attempts));
|
|
20965
|
+
}
|
|
20966
|
+
}
|
|
20967
|
+
} catch (error) {
|
|
20968
|
+
lastError = error.message;
|
|
20969
|
+
if (attempts < this.config.retryAttempts) {
|
|
20970
|
+
await new Promise((resolve) => setTimeout(resolve, this.config.retryDelay * attempts));
|
|
20971
|
+
} else {
|
|
20972
|
+
this.emit("replication.retry.exhausted", {
|
|
20973
|
+
attempts,
|
|
20974
|
+
lastError,
|
|
20975
|
+
item
|
|
20976
|
+
});
|
|
20977
|
+
}
|
|
20978
|
+
}
|
|
20979
|
+
}
|
|
20980
|
+
this.stats.failedOperations++;
|
|
20981
|
+
this.emit("replication.failed", {
|
|
20982
|
+
attempts,
|
|
20983
|
+
lastError,
|
|
20984
|
+
item,
|
|
20985
|
+
stats: this.stats
|
|
20986
|
+
});
|
|
20987
|
+
}
|
|
20988
|
+
// Utility methods
|
|
20989
|
+
async getReplicationStats() {
|
|
20990
|
+
const replicatorStats = await Promise.all(
|
|
20991
|
+
this.replicators.map(async (replicator) => {
|
|
20992
|
+
const status = await replicator.instance.getStatus();
|
|
20993
|
+
return {
|
|
20994
|
+
id: replicator.id,
|
|
20995
|
+
driver: replicator.driver,
|
|
20996
|
+
config: replicator.config,
|
|
20997
|
+
status
|
|
20998
|
+
};
|
|
20999
|
+
})
|
|
21000
|
+
);
|
|
21001
|
+
return {
|
|
21002
|
+
enabled: this.config.enabled,
|
|
21003
|
+
replicators: replicatorStats,
|
|
21004
|
+
queue: {
|
|
21005
|
+
length: this.queue.length,
|
|
21006
|
+
isProcessing: this.isProcessing
|
|
21007
|
+
},
|
|
21008
|
+
stats: this.stats,
|
|
21009
|
+
lastSync: this.stats.lastSync
|
|
17944
21010
|
};
|
|
17945
21011
|
}
|
|
21012
|
+
async getReplicationLogs(options = {}) {
|
|
21013
|
+
if (!this.replicationLog) {
|
|
21014
|
+
return [];
|
|
21015
|
+
}
|
|
21016
|
+
const {
|
|
21017
|
+
resourceName,
|
|
21018
|
+
operation,
|
|
21019
|
+
status,
|
|
21020
|
+
limit = 100,
|
|
21021
|
+
offset = 0
|
|
21022
|
+
} = options;
|
|
21023
|
+
let query = {};
|
|
21024
|
+
if (resourceName) {
|
|
21025
|
+
query.resourceName = resourceName;
|
|
21026
|
+
}
|
|
21027
|
+
if (operation) {
|
|
21028
|
+
query.operation = operation;
|
|
21029
|
+
}
|
|
21030
|
+
if (status) {
|
|
21031
|
+
query.status = status;
|
|
21032
|
+
}
|
|
21033
|
+
const logs = await this.replicationLog.list(query);
|
|
21034
|
+
return logs.slice(offset, offset + limit);
|
|
21035
|
+
}
|
|
21036
|
+
async retryFailedReplications() {
|
|
21037
|
+
if (!this.replicationLog) {
|
|
21038
|
+
return { retried: 0 };
|
|
21039
|
+
}
|
|
21040
|
+
const failedLogs = await this.replicationLog.list({
|
|
21041
|
+
status: "failed"
|
|
21042
|
+
});
|
|
21043
|
+
let retried = 0;
|
|
21044
|
+
for (const log of failedLogs) {
|
|
21045
|
+
try {
|
|
21046
|
+
await this.queueReplication(
|
|
21047
|
+
log.resourceName,
|
|
21048
|
+
log.operation,
|
|
21049
|
+
log.recordId,
|
|
21050
|
+
log.data
|
|
21051
|
+
);
|
|
21052
|
+
retried++;
|
|
21053
|
+
} catch (error) {
|
|
21054
|
+
console.error("Failed to retry replication:", error);
|
|
21055
|
+
}
|
|
21056
|
+
}
|
|
21057
|
+
return { retried };
|
|
21058
|
+
}
|
|
21059
|
+
async syncAllData(replicatorId) {
|
|
21060
|
+
const replicator = this.replicators.find((r) => r.id === replicatorId);
|
|
21061
|
+
if (!replicator) {
|
|
21062
|
+
throw new Error(`Replicator not found: ${replicatorId}`);
|
|
21063
|
+
}
|
|
21064
|
+
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
21065
|
+
for (const resourceName in this.database.resources) {
|
|
21066
|
+
if (resourceName === "replication_logs") continue;
|
|
21067
|
+
if (replicator.instance.shouldReplicateResource(resourceName)) {
|
|
21068
|
+
this.emit("replication.sync.resource", { resourceName, replicatorId });
|
|
21069
|
+
const resource = this.database.resources[resourceName];
|
|
21070
|
+
const allRecords = await resource.getAll();
|
|
21071
|
+
for (const record of allRecords) {
|
|
21072
|
+
await replicator.instance.replicate(resourceName, "insert", record, record.id);
|
|
21073
|
+
}
|
|
21074
|
+
}
|
|
21075
|
+
}
|
|
21076
|
+
this.emit("replication.sync.completed", { replicatorId, stats: this.stats });
|
|
21077
|
+
}
|
|
17946
21078
|
}
|
|
17947
21079
|
|
|
17948
21080
|
exports.AVAILABLE_BEHAVIORS = AVAILABLE_BEHAVIORS;
|
|
21081
|
+
exports.AuditPlugin = AuditPlugin;
|
|
17949
21082
|
exports.AuthenticationError = AuthenticationError;
|
|
17950
21083
|
exports.BaseError = BaseError;
|
|
17951
21084
|
exports.Cache = Cache;
|
|
@@ -17958,8 +21091,10 @@ exports.Database = Database;
|
|
|
17958
21091
|
exports.DatabaseError = DatabaseError;
|
|
17959
21092
|
exports.EncryptionError = EncryptionError;
|
|
17960
21093
|
exports.ErrorMap = ErrorMap;
|
|
21094
|
+
exports.FullTextPlugin = FullTextPlugin;
|
|
17961
21095
|
exports.InvalidResourceItem = InvalidResourceItem;
|
|
17962
21096
|
exports.MemoryCache = MemoryCache;
|
|
21097
|
+
exports.MetricsPlugin = MetricsPlugin;
|
|
17963
21098
|
exports.MissingMetadata = MissingMetadata;
|
|
17964
21099
|
exports.NoSuchBucket = NoSuchBucket;
|
|
17965
21100
|
exports.NoSuchKey = NoSuchKey;
|
|
@@ -17967,6 +21102,7 @@ exports.NotFound = NotFound;
|
|
|
17967
21102
|
exports.PermissionError = PermissionError;
|
|
17968
21103
|
exports.Plugin = Plugin;
|
|
17969
21104
|
exports.PluginObject = PluginObject;
|
|
21105
|
+
exports.ReplicationPlugin = ReplicationPlugin;
|
|
17970
21106
|
exports.Resource = Resource;
|
|
17971
21107
|
exports.ResourceIdsPageReader = ResourceIdsPageReader;
|
|
17972
21108
|
exports.ResourceIdsReader = ResourceIdsReader;
|