s3db.js 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +620 -1222
- package/dist/s3db.cjs.js +3638 -502
- package/dist/s3db.cjs.min.js +1 -31
- package/dist/s3db.d.ts +891 -62
- package/dist/s3db.es.js +3636 -504
- package/dist/s3db.es.min.js +1 -31
- package/dist/s3db.iife.js +3638 -502
- package/dist/s3db.iife.min.js +1 -31
- package/package.json +33 -14
package/dist/s3db.iife.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/* istanbul ignore file */
|
|
2
1
|
var S3DB = (function (exports, nanoid, lodashEs, promisePool, clientS3, crypto, flat, FastestValidator, web) {
|
|
3
2
|
'use strict';
|
|
4
3
|
|
|
@@ -933,15 +932,24 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
933
932
|
if (errorClass) return new errorClass(data);
|
|
934
933
|
return error;
|
|
935
934
|
}
|
|
936
|
-
async putObject({ key, metadata, contentType, body, contentEncoding }) {
|
|
935
|
+
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength }) {
|
|
936
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
937
|
+
const stringMetadata = {};
|
|
938
|
+
if (metadata) {
|
|
939
|
+
for (const [k, v] of Object.entries(metadata)) {
|
|
940
|
+
const validKey = String(k).replace(/[^a-zA-Z0-9\-_]/g, "_");
|
|
941
|
+
stringMetadata[validKey] = String(v);
|
|
942
|
+
}
|
|
943
|
+
}
|
|
937
944
|
const options2 = {
|
|
938
945
|
Bucket: this.config.bucket,
|
|
939
|
-
Key:
|
|
940
|
-
Metadata:
|
|
946
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key,
|
|
947
|
+
Metadata: stringMetadata,
|
|
941
948
|
Body: body || Buffer.alloc(0)
|
|
942
949
|
};
|
|
943
950
|
if (contentType !== void 0) options2.ContentType = contentType;
|
|
944
951
|
if (contentEncoding !== void 0) options2.ContentEncoding = contentEncoding;
|
|
952
|
+
if (contentLength !== void 0) options2.ContentLength = contentLength;
|
|
945
953
|
try {
|
|
946
954
|
const response = await this.sendCommand(new clientS3.PutObjectCommand(options2));
|
|
947
955
|
this.emit("putObject", response, options2);
|
|
@@ -954,9 +962,10 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
954
962
|
}
|
|
955
963
|
}
|
|
956
964
|
async getObject(key) {
|
|
965
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
957
966
|
const options2 = {
|
|
958
967
|
Bucket: this.config.bucket,
|
|
959
|
-
Key: path.join(
|
|
968
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
960
969
|
};
|
|
961
970
|
try {
|
|
962
971
|
const response = await this.sendCommand(new clientS3.GetObjectCommand(options2));
|
|
@@ -970,9 +979,10 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
970
979
|
}
|
|
971
980
|
}
|
|
972
981
|
async headObject(key) {
|
|
982
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
973
983
|
const options2 = {
|
|
974
984
|
Bucket: this.config.bucket,
|
|
975
|
-
Key:
|
|
985
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
976
986
|
};
|
|
977
987
|
try {
|
|
978
988
|
const response = await this.sendCommand(new clientS3.HeadObjectCommand(options2));
|
|
@@ -1014,9 +1024,10 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
1014
1024
|
}
|
|
1015
1025
|
}
|
|
1016
1026
|
async deleteObject(key) {
|
|
1027
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
1017
1028
|
const options2 = {
|
|
1018
1029
|
Bucket: this.config.bucket,
|
|
1019
|
-
Key:
|
|
1030
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
1020
1031
|
};
|
|
1021
1032
|
try {
|
|
1022
1033
|
const response = await this.sendCommand(new clientS3.DeleteObjectCommand(options2));
|
|
@@ -1030,13 +1041,14 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
1030
1041
|
}
|
|
1031
1042
|
}
|
|
1032
1043
|
async deleteObjects(keys) {
|
|
1044
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
1033
1045
|
const packages = lodashEs.chunk(keys, 1e3);
|
|
1034
1046
|
const { results, errors } = await promisePool.PromisePool.for(packages).withConcurrency(this.parallelism).process(async (keys2) => {
|
|
1035
1047
|
const options2 = {
|
|
1036
1048
|
Bucket: this.config.bucket,
|
|
1037
1049
|
Delete: {
|
|
1038
1050
|
Objects: keys2.map((key) => ({
|
|
1039
|
-
Key:
|
|
1051
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
1040
1052
|
}))
|
|
1041
1053
|
}
|
|
1042
1054
|
};
|
|
@@ -1064,12 +1076,13 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
1064
1076
|
* @returns {Promise<number>} Number of objects deleted
|
|
1065
1077
|
*/
|
|
1066
1078
|
async deleteAll({ prefix } = {}) {
|
|
1079
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
1067
1080
|
let continuationToken;
|
|
1068
1081
|
let totalDeleted = 0;
|
|
1069
1082
|
do {
|
|
1070
1083
|
const listCommand = new clientS3.ListObjectsV2Command({
|
|
1071
1084
|
Bucket: this.config.bucket,
|
|
1072
|
-
Prefix:
|
|
1085
|
+
Prefix: keyPrefix ? path.join(keyPrefix, prefix) : prefix,
|
|
1073
1086
|
ContinuationToken: continuationToken
|
|
1074
1087
|
});
|
|
1075
1088
|
const listResponse = await this.client.send(listCommand);
|
|
@@ -1211,6 +1224,10 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
1211
1224
|
prefix,
|
|
1212
1225
|
offset
|
|
1213
1226
|
});
|
|
1227
|
+
if (!continuationToken) {
|
|
1228
|
+
this.emit("getKeysPage", [], params);
|
|
1229
|
+
return [];
|
|
1230
|
+
}
|
|
1214
1231
|
}
|
|
1215
1232
|
while (truncated) {
|
|
1216
1233
|
const options2 = {
|
|
@@ -1223,8 +1240,8 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
1223
1240
|
}
|
|
1224
1241
|
truncated = res.IsTruncated || false;
|
|
1225
1242
|
continuationToken = res.NextContinuationToken;
|
|
1226
|
-
if (keys.length
|
|
1227
|
-
keys = keys.
|
|
1243
|
+
if (keys.length >= amount) {
|
|
1244
|
+
keys = keys.slice(0, amount);
|
|
1228
1245
|
break;
|
|
1229
1246
|
}
|
|
1230
1247
|
}
|
|
@@ -5996,6 +6013,16 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
5996
6013
|
};
|
|
5997
6014
|
}
|
|
5998
6015
|
|
|
6016
|
+
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
6017
|
+
function getOwnPropertyDescriptors(obj) {
|
|
6018
|
+
var keys = Object.keys(obj);
|
|
6019
|
+
var descriptors = {};
|
|
6020
|
+
for (var i = 0; i < keys.length; i++) {
|
|
6021
|
+
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
6022
|
+
}
|
|
6023
|
+
return descriptors;
|
|
6024
|
+
};
|
|
6025
|
+
|
|
5999
6026
|
var formatRegExp = /%[sdj%]/g;
|
|
6000
6027
|
function format(f) {
|
|
6001
6028
|
if (!isString(f)) {
|
|
@@ -6481,6 +6508,64 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
6481
6508
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
6482
6509
|
}
|
|
6483
6510
|
|
|
6511
|
+
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
6512
|
+
|
|
6513
|
+
function promisify(original) {
|
|
6514
|
+
if (typeof original !== 'function')
|
|
6515
|
+
throw new TypeError('The "original" argument must be of type Function');
|
|
6516
|
+
|
|
6517
|
+
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
6518
|
+
var fn = original[kCustomPromisifiedSymbol];
|
|
6519
|
+
if (typeof fn !== 'function') {
|
|
6520
|
+
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
6521
|
+
}
|
|
6522
|
+
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
6523
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
6524
|
+
});
|
|
6525
|
+
return fn;
|
|
6526
|
+
}
|
|
6527
|
+
|
|
6528
|
+
function fn() {
|
|
6529
|
+
var promiseResolve, promiseReject;
|
|
6530
|
+
var promise = new Promise(function (resolve, reject) {
|
|
6531
|
+
promiseResolve = resolve;
|
|
6532
|
+
promiseReject = reject;
|
|
6533
|
+
});
|
|
6534
|
+
|
|
6535
|
+
var args = [];
|
|
6536
|
+
for (var i = 0; i < arguments.length; i++) {
|
|
6537
|
+
args.push(arguments[i]);
|
|
6538
|
+
}
|
|
6539
|
+
args.push(function (err, value) {
|
|
6540
|
+
if (err) {
|
|
6541
|
+
promiseReject(err);
|
|
6542
|
+
} else {
|
|
6543
|
+
promiseResolve(value);
|
|
6544
|
+
}
|
|
6545
|
+
});
|
|
6546
|
+
|
|
6547
|
+
try {
|
|
6548
|
+
original.apply(this, args);
|
|
6549
|
+
} catch (err) {
|
|
6550
|
+
promiseReject(err);
|
|
6551
|
+
}
|
|
6552
|
+
|
|
6553
|
+
return promise;
|
|
6554
|
+
}
|
|
6555
|
+
|
|
6556
|
+
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
6557
|
+
|
|
6558
|
+
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
6559
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
6560
|
+
});
|
|
6561
|
+
return Object.defineProperties(
|
|
6562
|
+
fn,
|
|
6563
|
+
getOwnPropertyDescriptors(original)
|
|
6564
|
+
);
|
|
6565
|
+
}
|
|
6566
|
+
|
|
6567
|
+
promisify.custom = kCustomPromisifiedSymbol;
|
|
6568
|
+
|
|
6484
6569
|
function BufferList() {
|
|
6485
6570
|
this.head = null;
|
|
6486
6571
|
this.tail = null;
|
|
@@ -8710,41 +8795,41 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8710
8795
|
}
|
|
8711
8796
|
|
|
8712
8797
|
const S3_METADATA_LIMIT_BYTES = 2048;
|
|
8713
|
-
async function handleInsert$
|
|
8798
|
+
async function handleInsert$4({ resource, data, mappedData }) {
|
|
8714
8799
|
const totalSize = calculateTotalSize(mappedData);
|
|
8715
8800
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8716
8801
|
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
8717
8802
|
}
|
|
8718
8803
|
return { mappedData, body: "" };
|
|
8719
8804
|
}
|
|
8720
|
-
async function handleUpdate$
|
|
8805
|
+
async function handleUpdate$4({ resource, id, data, mappedData }) {
|
|
8721
8806
|
const totalSize = calculateTotalSize(mappedData);
|
|
8722
8807
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8723
8808
|
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
8724
8809
|
}
|
|
8725
8810
|
return { mappedData, body: "" };
|
|
8726
8811
|
}
|
|
8727
|
-
async function handleUpsert$
|
|
8812
|
+
async function handleUpsert$4({ resource, id, data, mappedData }) {
|
|
8728
8813
|
const totalSize = calculateTotalSize(mappedData);
|
|
8729
8814
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8730
8815
|
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
8731
8816
|
}
|
|
8732
8817
|
return { mappedData, body: "" };
|
|
8733
8818
|
}
|
|
8734
|
-
async function handleGet$
|
|
8819
|
+
async function handleGet$4({ resource, metadata, body }) {
|
|
8735
8820
|
return { metadata, body };
|
|
8736
8821
|
}
|
|
8737
8822
|
|
|
8738
8823
|
var enforceLimits = /*#__PURE__*/Object.freeze({
|
|
8739
8824
|
__proto__: null,
|
|
8740
8825
|
S3_METADATA_LIMIT_BYTES: S3_METADATA_LIMIT_BYTES,
|
|
8741
|
-
handleGet: handleGet$
|
|
8742
|
-
handleInsert: handleInsert$
|
|
8743
|
-
handleUpdate: handleUpdate$
|
|
8744
|
-
handleUpsert: handleUpsert$
|
|
8826
|
+
handleGet: handleGet$4,
|
|
8827
|
+
handleInsert: handleInsert$4,
|
|
8828
|
+
handleUpdate: handleUpdate$4,
|
|
8829
|
+
handleUpsert: handleUpsert$4
|
|
8745
8830
|
});
|
|
8746
8831
|
|
|
8747
|
-
async function handleInsert$
|
|
8832
|
+
async function handleInsert$3({ resource, data, mappedData }) {
|
|
8748
8833
|
const totalSize = calculateTotalSize(mappedData);
|
|
8749
8834
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8750
8835
|
resource.emit("exceedsLimit", {
|
|
@@ -8757,7 +8842,7 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8757
8842
|
}
|
|
8758
8843
|
return { mappedData, body: "" };
|
|
8759
8844
|
}
|
|
8760
|
-
async function handleUpdate$
|
|
8845
|
+
async function handleUpdate$3({ resource, id, data, mappedData }) {
|
|
8761
8846
|
const totalSize = calculateTotalSize(mappedData);
|
|
8762
8847
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8763
8848
|
resource.emit("exceedsLimit", {
|
|
@@ -8771,7 +8856,7 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8771
8856
|
}
|
|
8772
8857
|
return { mappedData, body: "" };
|
|
8773
8858
|
}
|
|
8774
|
-
async function handleUpsert$
|
|
8859
|
+
async function handleUpsert$3({ resource, id, data, mappedData }) {
|
|
8775
8860
|
const totalSize = calculateTotalSize(mappedData);
|
|
8776
8861
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8777
8862
|
resource.emit("exceedsLimit", {
|
|
@@ -8785,30 +8870,30 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8785
8870
|
}
|
|
8786
8871
|
return { mappedData, body: "" };
|
|
8787
8872
|
}
|
|
8788
|
-
async function handleGet$
|
|
8873
|
+
async function handleGet$3({ resource, metadata, body }) {
|
|
8789
8874
|
return { metadata, body };
|
|
8790
8875
|
}
|
|
8791
8876
|
|
|
8792
|
-
var
|
|
8877
|
+
var userManaged = /*#__PURE__*/Object.freeze({
|
|
8793
8878
|
__proto__: null,
|
|
8794
|
-
handleGet: handleGet$
|
|
8795
|
-
handleInsert: handleInsert$
|
|
8796
|
-
handleUpdate: handleUpdate$
|
|
8797
|
-
handleUpsert: handleUpsert$
|
|
8879
|
+
handleGet: handleGet$3,
|
|
8880
|
+
handleInsert: handleInsert$3,
|
|
8881
|
+
handleUpdate: handleUpdate$3,
|
|
8882
|
+
handleUpsert: handleUpsert$3
|
|
8798
8883
|
});
|
|
8799
8884
|
|
|
8800
8885
|
const TRUNCATE_SUFFIX = "...";
|
|
8801
8886
|
const TRUNCATE_SUFFIX_BYTES = calculateUTF8Bytes(TRUNCATE_SUFFIX);
|
|
8802
|
-
async function handleInsert$
|
|
8887
|
+
async function handleInsert$2({ resource, data, mappedData }) {
|
|
8803
8888
|
return handleTruncate({ resource, data, mappedData });
|
|
8804
8889
|
}
|
|
8805
|
-
async function handleUpdate$
|
|
8890
|
+
async function handleUpdate$2({ resource, id, data, mappedData }) {
|
|
8806
8891
|
return handleTruncate({ resource, data, mappedData });
|
|
8807
8892
|
}
|
|
8808
|
-
async function handleUpsert$
|
|
8893
|
+
async function handleUpsert$2({ resource, id, data, mappedData }) {
|
|
8809
8894
|
return handleTruncate({ resource, data, mappedData });
|
|
8810
8895
|
}
|
|
8811
|
-
async function handleGet$
|
|
8896
|
+
async function handleGet$2({ resource, metadata, body }) {
|
|
8812
8897
|
return { metadata, body };
|
|
8813
8898
|
}
|
|
8814
8899
|
function handleTruncate({ resource, data, mappedData }) {
|
|
@@ -8848,25 +8933,25 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8848
8933
|
|
|
8849
8934
|
var dataTruncate = /*#__PURE__*/Object.freeze({
|
|
8850
8935
|
__proto__: null,
|
|
8851
|
-
handleGet: handleGet$
|
|
8852
|
-
handleInsert: handleInsert$
|
|
8853
|
-
handleUpdate: handleUpdate$
|
|
8854
|
-
handleUpsert: handleUpsert$
|
|
8936
|
+
handleGet: handleGet$2,
|
|
8937
|
+
handleInsert: handleInsert$2,
|
|
8938
|
+
handleUpdate: handleUpdate$2,
|
|
8939
|
+
handleUpsert: handleUpsert$2
|
|
8855
8940
|
});
|
|
8856
8941
|
|
|
8857
8942
|
const OVERFLOW_FLAG = "$overflow";
|
|
8858
8943
|
const OVERFLOW_FLAG_VALUE = "true";
|
|
8859
8944
|
const OVERFLOW_FLAG_BYTES = calculateUTF8Bytes(OVERFLOW_FLAG) + calculateUTF8Bytes(OVERFLOW_FLAG_VALUE);
|
|
8860
|
-
async function handleInsert({ resource, data, mappedData }) {
|
|
8945
|
+
async function handleInsert$1({ resource, data, mappedData }) {
|
|
8861
8946
|
return handleOverflow({ resource, data, mappedData });
|
|
8862
8947
|
}
|
|
8863
|
-
async function handleUpdate({ resource, id, data, mappedData }) {
|
|
8948
|
+
async function handleUpdate$1({ resource, id, data, mappedData }) {
|
|
8864
8949
|
return handleOverflow({ resource, data, mappedData });
|
|
8865
8950
|
}
|
|
8866
|
-
async function handleUpsert({ resource, id, data, mappedData }) {
|
|
8951
|
+
async function handleUpsert$1({ resource, id, data, mappedData }) {
|
|
8867
8952
|
return handleOverflow({ resource, data, mappedData });
|
|
8868
8953
|
}
|
|
8869
|
-
async function handleGet({ resource, metadata, body }) {
|
|
8954
|
+
async function handleGet$1({ resource, metadata, body }) {
|
|
8870
8955
|
if (metadata[OVERFLOW_FLAG] === OVERFLOW_FLAG_VALUE) {
|
|
8871
8956
|
try {
|
|
8872
8957
|
const bodyData = body ? JSON.parse(body) : {};
|
|
@@ -8908,6 +8993,51 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8908
8993
|
}
|
|
8909
8994
|
|
|
8910
8995
|
var bodyOverflow = /*#__PURE__*/Object.freeze({
|
|
8996
|
+
__proto__: null,
|
|
8997
|
+
handleGet: handleGet$1,
|
|
8998
|
+
handleInsert: handleInsert$1,
|
|
8999
|
+
handleUpdate: handleUpdate$1,
|
|
9000
|
+
handleUpsert: handleUpsert$1
|
|
9001
|
+
});
|
|
9002
|
+
|
|
9003
|
+
async function handleInsert({ resource, data, mappedData }) {
|
|
9004
|
+
const bodyContent = JSON.stringify(mappedData);
|
|
9005
|
+
return {
|
|
9006
|
+
mappedData: {},
|
|
9007
|
+
body: bodyContent
|
|
9008
|
+
};
|
|
9009
|
+
}
|
|
9010
|
+
async function handleUpdate({ resource, id, data, mappedData }) {
|
|
9011
|
+
const bodyContent = JSON.stringify(mappedData);
|
|
9012
|
+
return {
|
|
9013
|
+
mappedData: {},
|
|
9014
|
+
body: bodyContent
|
|
9015
|
+
};
|
|
9016
|
+
}
|
|
9017
|
+
async function handleUpsert({ resource, id, data, mappedData }) {
|
|
9018
|
+
const bodyContent = JSON.stringify(mappedData);
|
|
9019
|
+
return {
|
|
9020
|
+
mappedData: {},
|
|
9021
|
+
body: bodyContent
|
|
9022
|
+
};
|
|
9023
|
+
}
|
|
9024
|
+
async function handleGet({ resource, metadata, body }) {
|
|
9025
|
+
try {
|
|
9026
|
+
const bodyData = body ? JSON.parse(body) : {};
|
|
9027
|
+
return {
|
|
9028
|
+
metadata: bodyData,
|
|
9029
|
+
body: ""
|
|
9030
|
+
};
|
|
9031
|
+
} catch (error) {
|
|
9032
|
+
console.warn(`Failed to parse body-only content:`, error.message);
|
|
9033
|
+
return {
|
|
9034
|
+
metadata,
|
|
9035
|
+
body: ""
|
|
9036
|
+
};
|
|
9037
|
+
}
|
|
9038
|
+
}
|
|
9039
|
+
|
|
9040
|
+
var bodyOnly = /*#__PURE__*/Object.freeze({
|
|
8911
9041
|
__proto__: null,
|
|
8912
9042
|
handleGet: handleGet,
|
|
8913
9043
|
handleInsert: handleInsert,
|
|
@@ -8916,10 +9046,11 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8916
9046
|
});
|
|
8917
9047
|
|
|
8918
9048
|
const behaviors = {
|
|
8919
|
-
"user-
|
|
9049
|
+
"user-managed": userManaged,
|
|
8920
9050
|
"enforce-limits": enforceLimits,
|
|
8921
9051
|
"data-truncate": dataTruncate,
|
|
8922
|
-
"body-overflow": bodyOverflow
|
|
9052
|
+
"body-overflow": bodyOverflow,
|
|
9053
|
+
"body-only": bodyOnly
|
|
8923
9054
|
};
|
|
8924
9055
|
function getBehavior(behaviorName) {
|
|
8925
9056
|
const behavior = behaviors[behaviorName];
|
|
@@ -8929,7 +9060,7 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8929
9060
|
return behavior;
|
|
8930
9061
|
}
|
|
8931
9062
|
const AVAILABLE_BEHAVIORS = Object.keys(behaviors);
|
|
8932
|
-
const DEFAULT_BEHAVIOR = "user-
|
|
9063
|
+
const DEFAULT_BEHAVIOR = "user-managed";
|
|
8933
9064
|
|
|
8934
9065
|
function createIdGeneratorWithSize(size) {
|
|
8935
9066
|
return nanoid.customAlphabet(nanoid.urlAlphabet, size);
|
|
@@ -8942,7 +9073,7 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8942
9073
|
* @param {Object} config.client - S3 client instance
|
|
8943
9074
|
* @param {string} [config.version='v0'] - Resource version
|
|
8944
9075
|
* @param {Object} [config.attributes={}] - Resource attributes schema
|
|
8945
|
-
* @param {string} [config.behavior='user-
|
|
9076
|
+
* @param {string} [config.behavior='user-managed'] - Resource behavior strategy
|
|
8946
9077
|
* @param {string} [config.passphrase='secret'] - Encryption passphrase
|
|
8947
9078
|
* @param {number} [config.parallelism=10] - Parallelism for bulk operations
|
|
8948
9079
|
* @param {Array} [config.observers=[]] - Observer instances
|
|
@@ -8956,6 +9087,7 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8956
9087
|
* @param {Object} [config.options={}] - Additional options
|
|
8957
9088
|
* @param {Function} [config.idGenerator] - Custom ID generator function
|
|
8958
9089
|
* @param {number} [config.idSize=22] - Size for auto-generated IDs
|
|
9090
|
+
* @param {boolean} [config.versioningEnabled=false] - Enable versioning for this resource
|
|
8959
9091
|
* @example
|
|
8960
9092
|
* const users = new Resource({
|
|
8961
9093
|
* name: 'users',
|
|
@@ -8965,7 +9097,7 @@ ${JSON.stringify(validation, null, 2)}`, {
|
|
|
8965
9097
|
* email: 'string|required',
|
|
8966
9098
|
* password: 'secret|required'
|
|
8967
9099
|
* },
|
|
8968
|
-
* behavior: 'user-
|
|
9100
|
+
* behavior: 'user-managed',
|
|
8969
9101
|
* passphrase: 'my-secret-key',
|
|
8970
9102
|
* timestamps: true,
|
|
8971
9103
|
* partitions: {
|
|
@@ -9029,7 +9161,8 @@ ${validation.errors.join("\n")}`);
|
|
|
9029
9161
|
allNestedObjectsOptional = true,
|
|
9030
9162
|
hooks = {},
|
|
9031
9163
|
idGenerator: customIdGenerator,
|
|
9032
|
-
idSize = 22
|
|
9164
|
+
idSize = 22,
|
|
9165
|
+
versioningEnabled = false
|
|
9033
9166
|
} = config;
|
|
9034
9167
|
this.name = name;
|
|
9035
9168
|
this.client = client;
|
|
@@ -9038,6 +9171,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9038
9171
|
this.observers = observers;
|
|
9039
9172
|
this.parallelism = parallelism;
|
|
9040
9173
|
this.passphrase = passphrase ?? "secret";
|
|
9174
|
+
this.versioningEnabled = versioningEnabled;
|
|
9041
9175
|
this.idGenerator = this.configureIdGenerator(customIdGenerator, idSize);
|
|
9042
9176
|
this.config = {
|
|
9043
9177
|
cache,
|
|
@@ -9145,6 +9279,15 @@ ${validation.errors.join("\n")}`);
|
|
|
9145
9279
|
}
|
|
9146
9280
|
}
|
|
9147
9281
|
this.setupPartitionHooks();
|
|
9282
|
+
if (this.versioningEnabled) {
|
|
9283
|
+
if (!this.config.partitions.byVersion) {
|
|
9284
|
+
this.config.partitions.byVersion = {
|
|
9285
|
+
fields: {
|
|
9286
|
+
_v: "string"
|
|
9287
|
+
}
|
|
9288
|
+
};
|
|
9289
|
+
}
|
|
9290
|
+
}
|
|
9148
9291
|
this.schema = new Schema({
|
|
9149
9292
|
name: this.name,
|
|
9150
9293
|
attributes: this.attributes,
|
|
@@ -9264,6 +9407,9 @@ ${validation.errors.join("\n")}`);
|
|
|
9264
9407
|
* @returns {boolean} True if field exists
|
|
9265
9408
|
*/
|
|
9266
9409
|
fieldExistsInAttributes(fieldName) {
|
|
9410
|
+
if (fieldName.startsWith("_")) {
|
|
9411
|
+
return true;
|
|
9412
|
+
}
|
|
9267
9413
|
if (!fieldName.includes(".")) {
|
|
9268
9414
|
return Object.keys(this.attributes || {}).includes(fieldName);
|
|
9269
9415
|
}
|
|
@@ -9317,12 +9463,12 @@ ${validation.errors.join("\n")}`);
|
|
|
9317
9463
|
return transformedValue;
|
|
9318
9464
|
}
|
|
9319
9465
|
/**
|
|
9320
|
-
* Get the main resource key (
|
|
9466
|
+
* Get the main resource key (new format without version in path)
|
|
9321
9467
|
* @param {string} id - Resource ID
|
|
9322
9468
|
* @returns {string} The main S3 key path
|
|
9323
9469
|
*/
|
|
9324
9470
|
getResourceKey(id) {
|
|
9325
|
-
return join(`resource=${this.name}`, `
|
|
9471
|
+
return join(`resource=${this.name}`, `data`, `id=${id}`);
|
|
9326
9472
|
}
|
|
9327
9473
|
/**
|
|
9328
9474
|
* Generate partition key for a resource in a specific partition
|
|
@@ -9391,12 +9537,23 @@ ${validation.errors.join("\n")}`);
|
|
|
9391
9537
|
}
|
|
9392
9538
|
return currentLevel;
|
|
9393
9539
|
}
|
|
9540
|
+
/**
|
|
9541
|
+
* Calculate estimated content length for body data
|
|
9542
|
+
* @param {string|Buffer} body - Body content
|
|
9543
|
+
* @returns {number} Estimated content length in bytes
|
|
9544
|
+
*/
|
|
9545
|
+
calculateContentLength(body) {
|
|
9546
|
+
if (!body) return 0;
|
|
9547
|
+
if (Buffer.isBuffer(body)) return body.length;
|
|
9548
|
+
if (typeof body === "string") return Buffer.byteLength(body, "utf8");
|
|
9549
|
+
if (typeof body === "object") return Buffer.byteLength(JSON.stringify(body), "utf8");
|
|
9550
|
+
return Buffer.byteLength(String(body), "utf8");
|
|
9551
|
+
}
|
|
9394
9552
|
/**
|
|
9395
9553
|
* Insert a new resource object
|
|
9396
|
-
* @param {Object}
|
|
9397
|
-
* @param {string} [
|
|
9398
|
-
* @
|
|
9399
|
-
* @returns {Promise<Object>} The inserted resource object with all attributes and generated ID
|
|
9554
|
+
* @param {Object} attributes - Resource attributes
|
|
9555
|
+
* @param {string} [attributes.id] - Custom ID (optional, auto-generated if not provided)
|
|
9556
|
+
* @returns {Promise<Object>} The created resource object with all attributes
|
|
9400
9557
|
* @example
|
|
9401
9558
|
* // Insert with auto-generated ID
|
|
9402
9559
|
* const user = await resource.insert({
|
|
@@ -9404,18 +9561,13 @@ ${validation.errors.join("\n")}`);
|
|
|
9404
9561
|
* email: 'john@example.com',
|
|
9405
9562
|
* age: 30
|
|
9406
9563
|
* });
|
|
9564
|
+
* console.log(user.id); // Auto-generated ID
|
|
9407
9565
|
*
|
|
9408
9566
|
* // Insert with custom ID
|
|
9409
9567
|
* const user = await resource.insert({
|
|
9410
|
-
* id: '
|
|
9411
|
-
* name: 'Jane Smith',
|
|
9412
|
-
* email: 'jane@example.com'
|
|
9413
|
-
* });
|
|
9414
|
-
*
|
|
9415
|
-
* // Insert with auto-generated password for secret field
|
|
9416
|
-
* const user = await resource.insert({
|
|
9568
|
+
* id: 'user-123',
|
|
9417
9569
|
* name: 'John Doe',
|
|
9418
|
-
* email: 'john@example.com'
|
|
9570
|
+
* email: 'john@example.com'
|
|
9419
9571
|
* });
|
|
9420
9572
|
*/
|
|
9421
9573
|
async insert({ id, ...attributes }) {
|
|
@@ -9423,7 +9575,8 @@ ${validation.errors.join("\n")}`);
|
|
|
9423
9575
|
attributes.createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9424
9576
|
attributes.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9425
9577
|
}
|
|
9426
|
-
const
|
|
9578
|
+
const completeData = { id, ...attributes };
|
|
9579
|
+
const preProcessedData = await this.executeHooks("preInsert", completeData);
|
|
9427
9580
|
const {
|
|
9428
9581
|
errors,
|
|
9429
9582
|
isValid,
|
|
@@ -9437,15 +9590,18 @@ ${validation.errors.join("\n")}`);
|
|
|
9437
9590
|
validation: errors
|
|
9438
9591
|
});
|
|
9439
9592
|
}
|
|
9440
|
-
|
|
9441
|
-
const
|
|
9593
|
+
const { id: validatedId, ...validatedAttributes } = validated;
|
|
9594
|
+
const finalId = validatedId || id || this.idGenerator();
|
|
9595
|
+
const mappedData = await this.schema.mapper(validatedAttributes);
|
|
9596
|
+
mappedData._v = String(this.version);
|
|
9442
9597
|
const behaviorImpl = getBehavior(this.behavior);
|
|
9443
9598
|
const { mappedData: processedMetadata, body } = await behaviorImpl.handleInsert({
|
|
9444
9599
|
resource: this,
|
|
9445
|
-
data:
|
|
9600
|
+
data: validatedAttributes,
|
|
9446
9601
|
mappedData
|
|
9447
9602
|
});
|
|
9448
|
-
const
|
|
9603
|
+
const finalMetadata = processedMetadata;
|
|
9604
|
+
const key = this.getResourceKey(finalId);
|
|
9449
9605
|
let contentType = void 0;
|
|
9450
9606
|
if (body && body !== "") {
|
|
9451
9607
|
try {
|
|
@@ -9455,12 +9611,13 @@ ${validation.errors.join("\n")}`);
|
|
|
9455
9611
|
}
|
|
9456
9612
|
}
|
|
9457
9613
|
await this.client.putObject({
|
|
9458
|
-
metadata:
|
|
9614
|
+
metadata: finalMetadata,
|
|
9459
9615
|
key,
|
|
9460
9616
|
body,
|
|
9461
|
-
contentType
|
|
9617
|
+
contentType,
|
|
9618
|
+
contentLength: this.calculateContentLength(body)
|
|
9462
9619
|
});
|
|
9463
|
-
const final = lodashEs.merge({ id },
|
|
9620
|
+
const final = lodashEs.merge({ id: finalId }, validatedAttributes);
|
|
9464
9621
|
await this.executeHooks("afterInsert", final);
|
|
9465
9622
|
this.emit("insert", final);
|
|
9466
9623
|
return final;
|
|
@@ -9479,7 +9636,8 @@ ${validation.errors.join("\n")}`);
|
|
|
9479
9636
|
const key = this.getResourceKey(id);
|
|
9480
9637
|
try {
|
|
9481
9638
|
const request = await this.client.headObject(key);
|
|
9482
|
-
const
|
|
9639
|
+
const objectVersionRaw = request.Metadata?._v || this.version;
|
|
9640
|
+
const objectVersion = typeof objectVersionRaw === "string" && objectVersionRaw.startsWith("v") ? objectVersionRaw.slice(1) : objectVersionRaw;
|
|
9483
9641
|
const schema = await this.getSchemaForVersion(objectVersion);
|
|
9484
9642
|
let metadata = await schema.unmapper(request.Metadata);
|
|
9485
9643
|
const behaviorImpl = getBehavior(this.behavior);
|
|
@@ -9504,9 +9662,13 @@ ${validation.errors.join("\n")}`);
|
|
|
9504
9662
|
data._lastModified = request.LastModified;
|
|
9505
9663
|
data._hasContent = request.ContentLength > 0;
|
|
9506
9664
|
data._mimeType = request.ContentType || null;
|
|
9665
|
+
data._v = objectVersion;
|
|
9507
9666
|
if (request.VersionId) data._versionId = request.VersionId;
|
|
9508
9667
|
if (request.Expiration) data._expiresAt = request.Expiration;
|
|
9509
9668
|
data._definitionHash = this.getDefinitionHash();
|
|
9669
|
+
if (objectVersion !== this.version) {
|
|
9670
|
+
data = await this.applyVersionMapping(data, objectVersion, this.version);
|
|
9671
|
+
}
|
|
9510
9672
|
this.emit("get", data);
|
|
9511
9673
|
return data;
|
|
9512
9674
|
} catch (error) {
|
|
@@ -9612,28 +9774,30 @@ ${validation.errors.join("\n")}`);
|
|
|
9612
9774
|
attributes.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9613
9775
|
}
|
|
9614
9776
|
const preProcessedData = await this.executeHooks("preUpdate", attributes);
|
|
9615
|
-
const
|
|
9616
|
-
|
|
9617
|
-
const { isValid, errors, data: validated } = await this.validate(attrs);
|
|
9777
|
+
const completeData = { ...originalData, ...preProcessedData, id };
|
|
9778
|
+
const { isValid, errors, data: validated } = await this.validate(completeData);
|
|
9618
9779
|
if (!isValid) {
|
|
9619
9780
|
throw new InvalidResourceItem({
|
|
9620
|
-
bucket: this.client.bucket,
|
|
9781
|
+
bucket: this.client.config.bucket,
|
|
9621
9782
|
resourceName: this.name,
|
|
9622
9783
|
attributes: preProcessedData,
|
|
9623
9784
|
validation: errors
|
|
9624
9785
|
});
|
|
9625
9786
|
}
|
|
9787
|
+
const { id: validatedId, ...validatedAttributes } = validated;
|
|
9626
9788
|
const oldData = { ...originalData, id };
|
|
9627
|
-
const newData = { ...
|
|
9789
|
+
const newData = { ...validatedAttributes, id };
|
|
9628
9790
|
await this.handlePartitionReferenceUpdates(oldData, newData);
|
|
9629
|
-
const mappedData = await this.schema.mapper(
|
|
9791
|
+
const mappedData = await this.schema.mapper(validatedAttributes);
|
|
9792
|
+
mappedData._v = String(this.version);
|
|
9630
9793
|
const behaviorImpl = getBehavior(this.behavior);
|
|
9631
9794
|
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
9632
9795
|
resource: this,
|
|
9633
9796
|
id,
|
|
9634
|
-
data:
|
|
9797
|
+
data: validatedAttributes,
|
|
9635
9798
|
mappedData
|
|
9636
9799
|
});
|
|
9800
|
+
const finalMetadata = processedMetadata;
|
|
9637
9801
|
const key = this.getResourceKey(id);
|
|
9638
9802
|
let existingContentType = void 0;
|
|
9639
9803
|
let finalBody = body;
|
|
@@ -9661,16 +9825,19 @@ ${validation.errors.join("\n")}`);
|
|
|
9661
9825
|
} catch {
|
|
9662
9826
|
}
|
|
9663
9827
|
}
|
|
9828
|
+
if (this.versioningEnabled && originalData._v !== this.version) {
|
|
9829
|
+
await this.createHistoricalVersion(id, originalData);
|
|
9830
|
+
}
|
|
9664
9831
|
await this.client.putObject({
|
|
9665
9832
|
key,
|
|
9666
9833
|
body: finalBody,
|
|
9667
9834
|
contentType: finalContentType,
|
|
9668
|
-
metadata:
|
|
9835
|
+
metadata: finalMetadata
|
|
9669
9836
|
});
|
|
9670
|
-
|
|
9671
|
-
await this.executeHooks("afterUpdate",
|
|
9672
|
-
this.emit("update", preProcessedData,
|
|
9673
|
-
return
|
|
9837
|
+
validatedAttributes.id = id;
|
|
9838
|
+
await this.executeHooks("afterUpdate", validatedAttributes);
|
|
9839
|
+
this.emit("update", preProcessedData, validatedAttributes);
|
|
9840
|
+
return validatedAttributes;
|
|
9674
9841
|
}
|
|
9675
9842
|
/**
|
|
9676
9843
|
* Delete a resource object by ID
|
|
@@ -9759,7 +9926,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9759
9926
|
prefix = `resource=${this.name}/partition=${partition}`;
|
|
9760
9927
|
}
|
|
9761
9928
|
} else {
|
|
9762
|
-
prefix = `resource=${this.name}/
|
|
9929
|
+
prefix = `resource=${this.name}/data`;
|
|
9763
9930
|
}
|
|
9764
9931
|
const count = await this.client.count({ prefix });
|
|
9765
9932
|
this.emit("count", count);
|
|
@@ -9828,7 +9995,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9828
9995
|
`deleteAll() is a dangerous operation and requires paranoid: false option. Current paranoid setting: ${this.config.paranoid}`
|
|
9829
9996
|
);
|
|
9830
9997
|
}
|
|
9831
|
-
const prefix = `resource=${this.name}/
|
|
9998
|
+
const prefix = `resource=${this.name}/data`;
|
|
9832
9999
|
const deletedCount = await this.client.deleteAll({ prefix });
|
|
9833
10000
|
this.emit("deleteAll", {
|
|
9834
10001
|
version: this.version,
|
|
@@ -9906,7 +10073,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9906
10073
|
prefix = `resource=${this.name}/partition=${partition}`;
|
|
9907
10074
|
}
|
|
9908
10075
|
} else {
|
|
9909
|
-
prefix = `resource=${this.name}/
|
|
10076
|
+
prefix = `resource=${this.name}/data`;
|
|
9910
10077
|
}
|
|
9911
10078
|
const keys = await this.client.getKeysPage({
|
|
9912
10079
|
prefix,
|
|
@@ -9923,154 +10090,175 @@ ${validation.errors.join("\n")}`);
|
|
|
9923
10090
|
return ids;
|
|
9924
10091
|
}
|
|
9925
10092
|
/**
|
|
9926
|
-
* List
|
|
10093
|
+
* List resources with optional partition filtering and pagination
|
|
9927
10094
|
* @param {Object} [params] - List parameters
|
|
9928
10095
|
* @param {string} [params.partition] - Partition name to list from
|
|
9929
10096
|
* @param {Object} [params.partitionValues] - Partition field values to filter by
|
|
9930
|
-
* @param {number} [params.limit] - Maximum number of results
|
|
9931
|
-
* @param {number} [params.offset=0] -
|
|
9932
|
-
* @returns {Promise<Object[]>} Array of resource objects
|
|
10097
|
+
* @param {number} [params.limit] - Maximum number of results
|
|
10098
|
+
* @param {number} [params.offset=0] - Number of results to skip
|
|
10099
|
+
* @returns {Promise<Object[]>} Array of resource objects
|
|
9933
10100
|
* @example
|
|
9934
10101
|
* // List all resources
|
|
9935
10102
|
* const allUsers = await resource.list();
|
|
9936
10103
|
*
|
|
9937
10104
|
* // List with pagination
|
|
9938
|
-
* const
|
|
9939
|
-
* const secondPage = await resource.list({ limit: 10, offset: 10 });
|
|
10105
|
+
* const first10 = await resource.list({ limit: 10, offset: 0 });
|
|
9940
10106
|
*
|
|
9941
10107
|
* // List from specific partition
|
|
9942
|
-
* const
|
|
9943
|
-
* partition: '
|
|
9944
|
-
* partitionValues: { '
|
|
9945
|
-
* });
|
|
9946
|
-
*
|
|
9947
|
-
* // List from partition with pagination
|
|
9948
|
-
* const googleUsersPage = await resource.list({
|
|
9949
|
-
* partition: 'byUtmSource',
|
|
9950
|
-
* partitionValues: { 'utm.source': 'google' },
|
|
9951
|
-
* limit: 5,
|
|
9952
|
-
* offset: 0
|
|
10108
|
+
* const usUsers = await resource.list({
|
|
10109
|
+
* partition: 'byCountry',
|
|
10110
|
+
* partitionValues: { 'profile.country': 'US' }
|
|
9953
10111
|
* });
|
|
9954
10112
|
*/
|
|
9955
10113
|
async list({ partition = null, partitionValues = {}, limit, offset = 0 } = {}) {
|
|
9956
10114
|
try {
|
|
9957
10115
|
if (!partition) {
|
|
9958
|
-
|
|
9959
|
-
try {
|
|
9960
|
-
ids2 = await this.listIds({ partition, partitionValues });
|
|
9961
|
-
} catch (listIdsError) {
|
|
9962
|
-
console.warn(`Failed to get list IDs:`, listIdsError.message);
|
|
9963
|
-
return [];
|
|
9964
|
-
}
|
|
9965
|
-
let filteredIds2 = ids2.slice(offset);
|
|
9966
|
-
if (limit) {
|
|
9967
|
-
filteredIds2 = filteredIds2.slice(0, limit);
|
|
9968
|
-
}
|
|
9969
|
-
const { results: results2, errors: errors2 } = await promisePool.PromisePool.for(filteredIds2).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
9970
|
-
console.warn(`Failed to get resource ${id}:`, error.message);
|
|
9971
|
-
return null;
|
|
9972
|
-
}).process(async (id) => {
|
|
9973
|
-
try {
|
|
9974
|
-
return await this.get(id);
|
|
9975
|
-
} catch (error) {
|
|
9976
|
-
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
9977
|
-
console.warn(`Decryption failed for ${id}, returning basic info`);
|
|
9978
|
-
return {
|
|
9979
|
-
id,
|
|
9980
|
-
_decryptionFailed: true,
|
|
9981
|
-
_error: error.message
|
|
9982
|
-
};
|
|
9983
|
-
}
|
|
9984
|
-
throw error;
|
|
9985
|
-
}
|
|
9986
|
-
});
|
|
9987
|
-
const validResults2 = results2.filter((item) => item !== null);
|
|
9988
|
-
this.emit("list", { partition, partitionValues, count: validResults2.length, errors: errors2.length });
|
|
9989
|
-
return validResults2;
|
|
9990
|
-
}
|
|
9991
|
-
if (!this.config.partitions || !this.config.partitions[partition]) {
|
|
9992
|
-
console.warn(`Partition '${partition}' not found in resource '${this.name}'`);
|
|
9993
|
-
this.emit("list", { partition, partitionValues, count: 0, errors: 0 });
|
|
9994
|
-
return [];
|
|
10116
|
+
return await this.listMain({ limit, offset });
|
|
9995
10117
|
}
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
|
|
10002
|
-
|
|
10003
|
-
|
|
10004
|
-
|
|
10118
|
+
return await this.listPartition({ partition, partitionValues, limit, offset });
|
|
10119
|
+
} catch (error) {
|
|
10120
|
+
return this.handleListError(error, { partition, partitionValues });
|
|
10121
|
+
}
|
|
10122
|
+
}
|
|
10123
|
+
/**
|
|
10124
|
+
* List resources from main resource (no partition)
|
|
10125
|
+
*/
|
|
10126
|
+
async listMain({ limit, offset = 0 }) {
|
|
10127
|
+
const ids = await this.listIds({ limit, offset });
|
|
10128
|
+
const results = await this.processListResults(ids, "main");
|
|
10129
|
+
this.emit("list", { count: results.length, errors: 0 });
|
|
10130
|
+
return results;
|
|
10131
|
+
}
|
|
10132
|
+
/**
|
|
10133
|
+
* List resources from specific partition
|
|
10134
|
+
*/
|
|
10135
|
+
async listPartition({ partition, partitionValues, limit, offset = 0 }) {
|
|
10136
|
+
if (!this.config.partitions?.[partition]) {
|
|
10137
|
+
console.warn(`Partition '${partition}' not found in resource '${this.name}'`);
|
|
10138
|
+
this.emit("list", { partition, partitionValues, count: 0, errors: 0 });
|
|
10139
|
+
return [];
|
|
10140
|
+
}
|
|
10141
|
+
const partitionDef = this.config.partitions[partition];
|
|
10142
|
+
const prefix = this.buildPartitionPrefix(partition, partitionDef, partitionValues);
|
|
10143
|
+
const keys = await this.client.getAllKeys({ prefix });
|
|
10144
|
+
const ids = this.extractIdsFromKeys(keys).slice(offset);
|
|
10145
|
+
const filteredIds = limit ? ids.slice(0, limit) : ids;
|
|
10146
|
+
const results = await this.processPartitionResults(filteredIds, partition, partitionDef, keys);
|
|
10147
|
+
this.emit("list", { partition, partitionValues, count: results.length, errors: 0 });
|
|
10148
|
+
return results;
|
|
10149
|
+
}
|
|
10150
|
+
/**
|
|
10151
|
+
* Build partition prefix from partition definition and values
|
|
10152
|
+
*/
|
|
10153
|
+
buildPartitionPrefix(partition, partitionDef, partitionValues) {
|
|
10154
|
+
const partitionSegments = [];
|
|
10155
|
+
const sortedFields = Object.entries(partitionDef.fields).sort(([a], [b]) => a.localeCompare(b));
|
|
10156
|
+
for (const [fieldName, rule] of sortedFields) {
|
|
10157
|
+
const value = partitionValues[fieldName];
|
|
10158
|
+
if (value !== void 0 && value !== null) {
|
|
10159
|
+
const transformedValue = this.applyPartitionRule(value, rule);
|
|
10160
|
+
partitionSegments.push(`${fieldName}=${transformedValue}`);
|
|
10005
10161
|
}
|
|
10006
|
-
|
|
10007
|
-
|
|
10008
|
-
|
|
10009
|
-
|
|
10010
|
-
|
|
10162
|
+
}
|
|
10163
|
+
if (partitionSegments.length > 0) {
|
|
10164
|
+
return `resource=${this.name}/partition=${partition}/${partitionSegments.join("/")}`;
|
|
10165
|
+
}
|
|
10166
|
+
return `resource=${this.name}/partition=${partition}`;
|
|
10167
|
+
}
|
|
10168
|
+
/**
|
|
10169
|
+
* Extract IDs from S3 keys
|
|
10170
|
+
*/
|
|
10171
|
+
extractIdsFromKeys(keys) {
|
|
10172
|
+
return keys.map((key) => {
|
|
10173
|
+
const parts = key.split("/");
|
|
10174
|
+
const idPart = parts.find((part) => part.startsWith("id="));
|
|
10175
|
+
return idPart ? idPart.replace("id=", "") : null;
|
|
10176
|
+
}).filter(Boolean);
|
|
10177
|
+
}
|
|
10178
|
+
/**
|
|
10179
|
+
* Process list results with error handling
|
|
10180
|
+
*/
|
|
10181
|
+
async processListResults(ids, context = "main") {
|
|
10182
|
+
const { results, errors } = await promisePool.PromisePool.for(ids).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
10183
|
+
console.warn(`Failed to get ${context} resource ${id}:`, error.message);
|
|
10184
|
+
return null;
|
|
10185
|
+
}).process(async (id) => {
|
|
10186
|
+
try {
|
|
10187
|
+
return await this.get(id);
|
|
10188
|
+
} catch (error) {
|
|
10189
|
+
return this.handleResourceError(error, id, context);
|
|
10011
10190
|
}
|
|
10012
|
-
|
|
10191
|
+
});
|
|
10192
|
+
return results.filter((item) => item !== null);
|
|
10193
|
+
}
|
|
10194
|
+
/**
|
|
10195
|
+
* Process partition results with error handling
|
|
10196
|
+
*/
|
|
10197
|
+
async processPartitionResults(ids, partition, partitionDef, keys) {
|
|
10198
|
+
const sortedFields = Object.entries(partitionDef.fields).sort(([a], [b]) => a.localeCompare(b));
|
|
10199
|
+
const { results, errors } = await promisePool.PromisePool.for(ids).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
10200
|
+
console.warn(`Failed to get partition resource ${id}:`, error.message);
|
|
10201
|
+
return null;
|
|
10202
|
+
}).process(async (id) => {
|
|
10013
10203
|
try {
|
|
10014
|
-
|
|
10015
|
-
|
|
10016
|
-
|
|
10017
|
-
|
|
10204
|
+
const actualPartitionValues = this.extractPartitionValuesFromKey(id, keys, sortedFields);
|
|
10205
|
+
return await this.getFromPartition({
|
|
10206
|
+
id,
|
|
10207
|
+
partitionName: partition,
|
|
10208
|
+
partitionValues: actualPartitionValues
|
|
10209
|
+
});
|
|
10210
|
+
} catch (error) {
|
|
10211
|
+
return this.handleResourceError(error, id, "partition");
|
|
10018
10212
|
}
|
|
10019
|
-
|
|
10020
|
-
|
|
10021
|
-
|
|
10022
|
-
|
|
10023
|
-
|
|
10024
|
-
|
|
10025
|
-
|
|
10026
|
-
|
|
10027
|
-
|
|
10028
|
-
|
|
10029
|
-
|
|
10030
|
-
|
|
10031
|
-
|
|
10032
|
-
|
|
10033
|
-
|
|
10034
|
-
|
|
10035
|
-
|
|
10036
|
-
|
|
10037
|
-
const keyParts = keyForId.split("/");
|
|
10038
|
-
const actualPartitionValues = {};
|
|
10039
|
-
for (const [fieldName, rule] of sortedFields) {
|
|
10040
|
-
const fieldPart = keyParts.find((part) => part.startsWith(`${fieldName}=`));
|
|
10041
|
-
if (fieldPart) {
|
|
10042
|
-
const value = fieldPart.replace(`${fieldName}=`, "");
|
|
10043
|
-
actualPartitionValues[fieldName] = value;
|
|
10044
|
-
}
|
|
10045
|
-
}
|
|
10046
|
-
return await this.getFromPartition({ id, partitionName: partition, partitionValues: actualPartitionValues });
|
|
10047
|
-
} catch (error) {
|
|
10048
|
-
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
10049
|
-
console.warn(`Decryption failed for partition resource ${id}, returning basic info`);
|
|
10050
|
-
return {
|
|
10051
|
-
id,
|
|
10052
|
-
_partition: partition,
|
|
10053
|
-
_partitionValues: partitionValues,
|
|
10054
|
-
_decryptionFailed: true,
|
|
10055
|
-
_error: error.message
|
|
10056
|
-
};
|
|
10057
|
-
}
|
|
10058
|
-
throw error;
|
|
10059
|
-
}
|
|
10060
|
-
});
|
|
10061
|
-
const validResults = results.filter((item) => item !== null);
|
|
10062
|
-
this.emit("list", { partition, partitionValues, count: validResults.length, errors: errors.length });
|
|
10063
|
-
return validResults;
|
|
10064
|
-
} catch (error) {
|
|
10065
|
-
if (error.message.includes("Partition '") && error.message.includes("' not found")) {
|
|
10066
|
-
console.warn(`Partition error in list method:`, error.message);
|
|
10067
|
-
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10068
|
-
return [];
|
|
10213
|
+
});
|
|
10214
|
+
return results.filter((item) => item !== null);
|
|
10215
|
+
}
|
|
10216
|
+
/**
|
|
10217
|
+
* Extract partition values from S3 key for specific ID
|
|
10218
|
+
*/
|
|
10219
|
+
extractPartitionValuesFromKey(id, keys, sortedFields) {
|
|
10220
|
+
const keyForId = keys.find((key) => key.includes(`id=${id}`));
|
|
10221
|
+
if (!keyForId) {
|
|
10222
|
+
throw new Error(`Partition key not found for ID ${id}`);
|
|
10223
|
+
}
|
|
10224
|
+
const keyParts = keyForId.split("/");
|
|
10225
|
+
const actualPartitionValues = {};
|
|
10226
|
+
for (const [fieldName] of sortedFields) {
|
|
10227
|
+
const fieldPart = keyParts.find((part) => part.startsWith(`${fieldName}=`));
|
|
10228
|
+
if (fieldPart) {
|
|
10229
|
+
const value = fieldPart.replace(`${fieldName}=`, "");
|
|
10230
|
+
actualPartitionValues[fieldName] = value;
|
|
10069
10231
|
}
|
|
10070
|
-
|
|
10232
|
+
}
|
|
10233
|
+
return actualPartitionValues;
|
|
10234
|
+
}
|
|
10235
|
+
/**
|
|
10236
|
+
* Handle resource-specific errors
|
|
10237
|
+
*/
|
|
10238
|
+
handleResourceError(error, id, context) {
|
|
10239
|
+
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
10240
|
+
console.warn(`Decryption failed for ${context} resource ${id}, returning basic info`);
|
|
10241
|
+
return {
|
|
10242
|
+
id,
|
|
10243
|
+
_decryptionFailed: true,
|
|
10244
|
+
_error: error.message,
|
|
10245
|
+
...context === "partition" && { _partition: context }
|
|
10246
|
+
};
|
|
10247
|
+
}
|
|
10248
|
+
throw error;
|
|
10249
|
+
}
|
|
10250
|
+
/**
|
|
10251
|
+
* Handle list method errors
|
|
10252
|
+
*/
|
|
10253
|
+
handleListError(error, { partition, partitionValues }) {
|
|
10254
|
+
if (error.message.includes("Partition '") && error.message.includes("' not found")) {
|
|
10255
|
+
console.warn(`Partition error in list method:`, error.message);
|
|
10071
10256
|
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10072
10257
|
return [];
|
|
10073
10258
|
}
|
|
10259
|
+
console.error(`Critical error in list method:`, error.message);
|
|
10260
|
+
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10261
|
+
return [];
|
|
10074
10262
|
}
|
|
10075
10263
|
/**
|
|
10076
10264
|
* Get multiple resources by their IDs
|
|
@@ -10418,30 +10606,14 @@ ${validation.errors.join("\n")}`);
|
|
|
10418
10606
|
for (const [partitionName, partition] of Object.entries(partitions)) {
|
|
10419
10607
|
const partitionKey = this.getPartitionKey({ partitionName, id: data.id, data });
|
|
10420
10608
|
if (partitionKey) {
|
|
10421
|
-
const mappedData = await this.schema.mapper(data);
|
|
10422
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10423
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleInsert({
|
|
10424
|
-
resource: this,
|
|
10425
|
-
data,
|
|
10426
|
-
mappedData
|
|
10427
|
-
});
|
|
10428
10609
|
const partitionMetadata = {
|
|
10429
|
-
|
|
10430
|
-
_version: this.version
|
|
10610
|
+
_v: String(this.version)
|
|
10431
10611
|
};
|
|
10432
|
-
let contentType = void 0;
|
|
10433
|
-
if (body && body !== "") {
|
|
10434
|
-
try {
|
|
10435
|
-
JSON.parse(body);
|
|
10436
|
-
contentType = "application/json";
|
|
10437
|
-
} catch {
|
|
10438
|
-
}
|
|
10439
|
-
}
|
|
10440
10612
|
await this.client.putObject({
|
|
10441
10613
|
key: partitionKey,
|
|
10442
10614
|
metadata: partitionMetadata,
|
|
10443
|
-
body,
|
|
10444
|
-
contentType
|
|
10615
|
+
body: "",
|
|
10616
|
+
contentType: void 0
|
|
10445
10617
|
});
|
|
10446
10618
|
}
|
|
10447
10619
|
}
|
|
@@ -10598,34 +10770,14 @@ ${validation.errors.join("\n")}`);
|
|
|
10598
10770
|
}
|
|
10599
10771
|
if (newPartitionKey) {
|
|
10600
10772
|
try {
|
|
10601
|
-
const mappedData = await this.schema.mapper(newData);
|
|
10602
|
-
if (mappedData.undefined !== void 0) delete mappedData.undefined;
|
|
10603
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10604
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
10605
|
-
resource: this,
|
|
10606
|
-
id,
|
|
10607
|
-
data: newData,
|
|
10608
|
-
mappedData
|
|
10609
|
-
});
|
|
10610
|
-
if (processedMetadata.undefined !== void 0) delete processedMetadata.undefined;
|
|
10611
10773
|
const partitionMetadata = {
|
|
10612
|
-
|
|
10613
|
-
_version: this.version
|
|
10774
|
+
_v: String(this.version)
|
|
10614
10775
|
};
|
|
10615
|
-
if (partitionMetadata.undefined !== void 0) delete partitionMetadata.undefined;
|
|
10616
|
-
let contentType = void 0;
|
|
10617
|
-
if (body && body !== "") {
|
|
10618
|
-
try {
|
|
10619
|
-
JSON.parse(body);
|
|
10620
|
-
contentType = "application/json";
|
|
10621
|
-
} catch {
|
|
10622
|
-
}
|
|
10623
|
-
}
|
|
10624
10776
|
await this.client.putObject({
|
|
10625
10777
|
key: newPartitionKey,
|
|
10626
10778
|
metadata: partitionMetadata,
|
|
10627
|
-
body,
|
|
10628
|
-
contentType
|
|
10779
|
+
body: "",
|
|
10780
|
+
contentType: void 0
|
|
10629
10781
|
});
|
|
10630
10782
|
} catch (error) {
|
|
10631
10783
|
console.warn(`New partition object could not be created for ${partitionName}:`, error.message);
|
|
@@ -10633,34 +10785,14 @@ ${validation.errors.join("\n")}`);
|
|
|
10633
10785
|
}
|
|
10634
10786
|
} else if (newPartitionKey) {
|
|
10635
10787
|
try {
|
|
10636
|
-
const mappedData = await this.schema.mapper(newData);
|
|
10637
|
-
if (mappedData.undefined !== void 0) delete mappedData.undefined;
|
|
10638
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10639
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
10640
|
-
resource: this,
|
|
10641
|
-
id,
|
|
10642
|
-
data: newData,
|
|
10643
|
-
mappedData
|
|
10644
|
-
});
|
|
10645
|
-
if (processedMetadata.undefined !== void 0) delete processedMetadata.undefined;
|
|
10646
10788
|
const partitionMetadata = {
|
|
10647
|
-
|
|
10648
|
-
_version: this.version
|
|
10789
|
+
_v: String(this.version)
|
|
10649
10790
|
};
|
|
10650
|
-
if (partitionMetadata.undefined !== void 0) delete partitionMetadata.undefined;
|
|
10651
|
-
let contentType = void 0;
|
|
10652
|
-
if (body && body !== "") {
|
|
10653
|
-
try {
|
|
10654
|
-
JSON.parse(body);
|
|
10655
|
-
contentType = "application/json";
|
|
10656
|
-
} catch {
|
|
10657
|
-
}
|
|
10658
|
-
}
|
|
10659
10791
|
await this.client.putObject({
|
|
10660
10792
|
key: newPartitionKey,
|
|
10661
10793
|
metadata: partitionMetadata,
|
|
10662
|
-
body,
|
|
10663
|
-
contentType
|
|
10794
|
+
body: "",
|
|
10795
|
+
contentType: void 0
|
|
10664
10796
|
});
|
|
10665
10797
|
} catch (error) {
|
|
10666
10798
|
console.warn(`Partition object could not be updated for ${partitionName}:`, error.message);
|
|
@@ -10683,32 +10815,15 @@ ${validation.errors.join("\n")}`);
|
|
|
10683
10815
|
}
|
|
10684
10816
|
const partitionKey = this.getPartitionKey({ partitionName, id: data.id, data });
|
|
10685
10817
|
if (partitionKey) {
|
|
10686
|
-
const mappedData = await this.schema.mapper(data);
|
|
10687
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10688
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
10689
|
-
resource: this,
|
|
10690
|
-
id: data.id,
|
|
10691
|
-
data,
|
|
10692
|
-
mappedData
|
|
10693
|
-
});
|
|
10694
10818
|
const partitionMetadata = {
|
|
10695
|
-
|
|
10696
|
-
_version: this.version
|
|
10819
|
+
_v: String(this.version)
|
|
10697
10820
|
};
|
|
10698
|
-
let contentType = void 0;
|
|
10699
|
-
if (body && body !== "") {
|
|
10700
|
-
try {
|
|
10701
|
-
JSON.parse(body);
|
|
10702
|
-
contentType = "application/json";
|
|
10703
|
-
} catch {
|
|
10704
|
-
}
|
|
10705
|
-
}
|
|
10706
10821
|
try {
|
|
10707
10822
|
await this.client.putObject({
|
|
10708
10823
|
key: partitionKey,
|
|
10709
10824
|
metadata: partitionMetadata,
|
|
10710
|
-
body,
|
|
10711
|
-
contentType
|
|
10825
|
+
body: "",
|
|
10826
|
+
contentType: void 0
|
|
10712
10827
|
});
|
|
10713
10828
|
} catch (error) {
|
|
10714
10829
|
console.warn(`Partition object could not be updated for ${partitionName}:`, error.message);
|
|
@@ -10758,40 +10873,75 @@ ${validation.errors.join("\n")}`);
|
|
|
10758
10873
|
throw new Error(`No partition values provided for partition '${partitionName}'`);
|
|
10759
10874
|
}
|
|
10760
10875
|
const partitionKey = join(`resource=${this.name}`, `partition=${partitionName}`, ...partitionSegments, `id=${id}`);
|
|
10761
|
-
|
|
10762
|
-
|
|
10763
|
-
|
|
10764
|
-
|
|
10765
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10766
|
-
let body = "";
|
|
10767
|
-
if (request.ContentLength > 0) {
|
|
10768
|
-
try {
|
|
10769
|
-
const fullObject = await this.client.getObject(partitionKey);
|
|
10770
|
-
body = await streamToString(fullObject.Body);
|
|
10771
|
-
} catch (error) {
|
|
10772
|
-
body = "";
|
|
10773
|
-
}
|
|
10876
|
+
try {
|
|
10877
|
+
await this.client.headObject(partitionKey);
|
|
10878
|
+
} catch (error) {
|
|
10879
|
+
throw new Error(`Resource with id '${id}' not found in partition '${partitionName}'`);
|
|
10774
10880
|
}
|
|
10775
|
-
const
|
|
10776
|
-
resource: this,
|
|
10777
|
-
metadata,
|
|
10778
|
-
body
|
|
10779
|
-
});
|
|
10780
|
-
let data = processedMetadata;
|
|
10781
|
-
data.id = id;
|
|
10782
|
-
data._contentLength = request.ContentLength;
|
|
10783
|
-
data._lastModified = request.LastModified;
|
|
10784
|
-
data._hasContent = request.ContentLength > 0;
|
|
10785
|
-
data._mimeType = request.ContentType || null;
|
|
10881
|
+
const data = await this.get(id);
|
|
10786
10882
|
data._partition = partitionName;
|
|
10787
10883
|
data._partitionValues = partitionValues;
|
|
10788
|
-
if (request.VersionId) data._versionId = request.VersionId;
|
|
10789
|
-
if (request.Expiration) data._expiresAt = request.Expiration;
|
|
10790
|
-
data._definitionHash = this.getDefinitionHash();
|
|
10791
|
-
if (data.undefined !== void 0) delete data.undefined;
|
|
10792
10884
|
this.emit("getFromPartition", data);
|
|
10793
10885
|
return data;
|
|
10794
10886
|
}
|
|
10887
|
+
/**
|
|
10888
|
+
* Create a historical version of an object
|
|
10889
|
+
* @param {string} id - Resource ID
|
|
10890
|
+
* @param {Object} data - Object data to store historically
|
|
10891
|
+
*/
|
|
10892
|
+
async createHistoricalVersion(id, data) {
|
|
10893
|
+
const historicalKey = join(`resource=${this.name}`, `historical`, `id=${id}`);
|
|
10894
|
+
const historicalData = {
|
|
10895
|
+
...data,
|
|
10896
|
+
_v: data._v || this.version,
|
|
10897
|
+
_historicalTimestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
10898
|
+
};
|
|
10899
|
+
const mappedData = await this.schema.mapper(historicalData);
|
|
10900
|
+
const behaviorImpl = getBehavior(this.behavior);
|
|
10901
|
+
const { mappedData: processedMetadata, body } = await behaviorImpl.handleInsert({
|
|
10902
|
+
resource: this,
|
|
10903
|
+
data: historicalData,
|
|
10904
|
+
mappedData
|
|
10905
|
+
});
|
|
10906
|
+
const finalMetadata = {
|
|
10907
|
+
...processedMetadata,
|
|
10908
|
+
_v: data._v || this.version,
|
|
10909
|
+
_historicalTimestamp: historicalData._historicalTimestamp
|
|
10910
|
+
};
|
|
10911
|
+
let contentType = void 0;
|
|
10912
|
+
if (body && body !== "") {
|
|
10913
|
+
try {
|
|
10914
|
+
JSON.parse(body);
|
|
10915
|
+
contentType = "application/json";
|
|
10916
|
+
} catch {
|
|
10917
|
+
}
|
|
10918
|
+
}
|
|
10919
|
+
await this.client.putObject({
|
|
10920
|
+
key: historicalKey,
|
|
10921
|
+
metadata: finalMetadata,
|
|
10922
|
+
body,
|
|
10923
|
+
contentType
|
|
10924
|
+
});
|
|
10925
|
+
}
|
|
10926
|
+
/**
|
|
10927
|
+
* Apply version mapping to convert an object from one version to another
|
|
10928
|
+
* @param {Object} data - Object data to map
|
|
10929
|
+
* @param {string} fromVersion - Source version
|
|
10930
|
+
* @param {string} toVersion - Target version
|
|
10931
|
+
* @returns {Object} Mapped object data
|
|
10932
|
+
*/
|
|
10933
|
+
async applyVersionMapping(data, fromVersion, toVersion) {
|
|
10934
|
+
if (fromVersion === toVersion) {
|
|
10935
|
+
return data;
|
|
10936
|
+
}
|
|
10937
|
+
const mappedData = {
|
|
10938
|
+
...data,
|
|
10939
|
+
_v: toVersion,
|
|
10940
|
+
_originalVersion: fromVersion,
|
|
10941
|
+
_versionMapped: true
|
|
10942
|
+
};
|
|
10943
|
+
return mappedData;
|
|
10944
|
+
}
|
|
10795
10945
|
}
|
|
10796
10946
|
function validateResourceConfig(config) {
|
|
10797
10947
|
const errors = [];
|
|
@@ -10906,7 +11056,7 @@ ${validation.errors.join("\n")}`);
|
|
|
10906
11056
|
this.version = "1";
|
|
10907
11057
|
this.s3dbVersion = (() => {
|
|
10908
11058
|
try {
|
|
10909
|
-
return true ? "
|
|
11059
|
+
return true ? "6.1.0" : "latest";
|
|
10910
11060
|
} catch (e) {
|
|
10911
11061
|
return "latest";
|
|
10912
11062
|
}
|
|
@@ -10917,8 +11067,10 @@ ${validation.errors.join("\n")}`);
|
|
|
10917
11067
|
this.verbose = options.verbose || false;
|
|
10918
11068
|
this.parallelism = parseInt(options.parallelism + "") || 10;
|
|
10919
11069
|
this.plugins = options.plugins || [];
|
|
11070
|
+
this.pluginList = options.plugins || [];
|
|
10920
11071
|
this.cache = options.cache;
|
|
10921
11072
|
this.passphrase = options.passphrase || "secret";
|
|
11073
|
+
this.versioningEnabled = options.versioningEnabled || false;
|
|
10922
11074
|
let connectionString = options.connectionString;
|
|
10923
11075
|
if (!connectionString && (options.bucket || options.accessKeyId || options.secretAccessKey)) {
|
|
10924
11076
|
const { bucket, region, accessKeyId, secretAccessKey, endpoint, forcePathStyle } = options;
|
|
@@ -10969,7 +11121,7 @@ ${validation.errors.join("\n")}`);
|
|
|
10969
11121
|
client: this.client,
|
|
10970
11122
|
version: currentVersion,
|
|
10971
11123
|
attributes: versionData.attributes,
|
|
10972
|
-
behavior: versionData.behavior || "user-
|
|
11124
|
+
behavior: versionData.behavior || "user-managed",
|
|
10973
11125
|
parallelism: this.parallelism,
|
|
10974
11126
|
passphrase: this.passphrase,
|
|
10975
11127
|
observers: [this],
|
|
@@ -10979,7 +11131,8 @@ ${validation.errors.join("\n")}`);
|
|
|
10979
11131
|
paranoid: versionData.paranoid !== void 0 ? versionData.paranoid : true,
|
|
10980
11132
|
allNestedObjectsOptional: versionData.allNestedObjectsOptional !== void 0 ? versionData.allNestedObjectsOptional : true,
|
|
10981
11133
|
autoDecrypt: versionData.autoDecrypt !== void 0 ? versionData.autoDecrypt : true,
|
|
10982
|
-
hooks: versionData.hooks || {}
|
|
11134
|
+
hooks: versionData.hooks || {},
|
|
11135
|
+
versioningEnabled: this.versioningEnabled
|
|
10983
11136
|
});
|
|
10984
11137
|
}
|
|
10985
11138
|
}
|
|
@@ -11054,7 +11207,7 @@ ${validation.errors.join("\n")}`);
|
|
|
11054
11207
|
}
|
|
11055
11208
|
const hashObj = {
|
|
11056
11209
|
attributes: stableAttributes,
|
|
11057
|
-
behavior: behavior || definition.behavior || "user-
|
|
11210
|
+
behavior: behavior || definition.behavior || "user-managed"
|
|
11058
11211
|
};
|
|
11059
11212
|
const stableString = jsonStableStringify(hashObj);
|
|
11060
11213
|
return `sha256:${crypto.createHash("sha256").update(stableString).digest("hex")}`;
|
|
@@ -11071,8 +11224,8 @@ ${validation.errors.join("\n")}`);
|
|
|
11071
11224
|
}
|
|
11072
11225
|
async startPlugins() {
|
|
11073
11226
|
const db = this;
|
|
11074
|
-
if (!lodashEs.isEmpty(this.
|
|
11075
|
-
const plugins = this.
|
|
11227
|
+
if (!lodashEs.isEmpty(this.pluginList)) {
|
|
11228
|
+
const plugins = this.pluginList.map((p) => lodashEs.isFunction(p) ? new p(this) : p);
|
|
11076
11229
|
const setupProms = plugins.map(async (plugin) => {
|
|
11077
11230
|
if (plugin.beforeSetup) await plugin.beforeSetup();
|
|
11078
11231
|
await plugin.setup(db);
|
|
@@ -11087,6 +11240,20 @@ ${validation.errors.join("\n")}`);
|
|
|
11087
11240
|
await Promise.all(startProms);
|
|
11088
11241
|
}
|
|
11089
11242
|
}
|
|
11243
|
+
/**
|
|
11244
|
+
* Register and setup a plugin
|
|
11245
|
+
* @param {Plugin} plugin - Plugin instance to register
|
|
11246
|
+
* @param {string} [name] - Optional name for the plugin (defaults to plugin.constructor.name)
|
|
11247
|
+
*/
|
|
11248
|
+
async usePlugin(plugin, name = null) {
|
|
11249
|
+
const pluginName = name || plugin.constructor.name.replace("Plugin", "").toLowerCase();
|
|
11250
|
+
this.plugins[pluginName] = plugin;
|
|
11251
|
+
if (this.isConnected()) {
|
|
11252
|
+
await plugin.setup(this);
|
|
11253
|
+
await plugin.start();
|
|
11254
|
+
}
|
|
11255
|
+
return plugin;
|
|
11256
|
+
}
|
|
11090
11257
|
async uploadMetadataFile() {
|
|
11091
11258
|
const metadata = {
|
|
11092
11259
|
version: this.version,
|
|
@@ -11117,7 +11284,7 @@ ${validation.errors.join("\n")}`);
|
|
|
11117
11284
|
[version]: {
|
|
11118
11285
|
hash: definitionHash,
|
|
11119
11286
|
attributes: resourceDef.attributes,
|
|
11120
|
-
behavior: resourceDef.behavior || "user-
|
|
11287
|
+
behavior: resourceDef.behavior || "user-managed",
|
|
11121
11288
|
timestamps: resource.config.timestamps,
|
|
11122
11289
|
partitions: resource.config.partitions,
|
|
11123
11290
|
paranoid: resource.config.paranoid,
|
|
@@ -11166,7 +11333,7 @@ ${validation.errors.join("\n")}`);
|
|
|
11166
11333
|
* @param {Object} [config.options] - Resource options (deprecated, use root level parameters)
|
|
11167
11334
|
* @returns {Object} Result with exists and hash information
|
|
11168
11335
|
*/
|
|
11169
|
-
resourceExistsWithSameHash({ name, attributes, behavior = "user-
|
|
11336
|
+
resourceExistsWithSameHash({ name, attributes, behavior = "user-managed", options = {} }) {
|
|
11170
11337
|
if (!this.resources[name]) {
|
|
11171
11338
|
return { exists: false, sameHash: false, hash: null };
|
|
11172
11339
|
}
|
|
@@ -11179,6 +11346,7 @@ ${validation.errors.join("\n")}`);
|
|
|
11179
11346
|
client: this.client,
|
|
11180
11347
|
version: existingResource.version,
|
|
11181
11348
|
passphrase: this.passphrase,
|
|
11349
|
+
versioningEnabled: this.versioningEnabled,
|
|
11182
11350
|
...options
|
|
11183
11351
|
});
|
|
11184
11352
|
const newHash = this.generateDefinitionHash(mockResource.export());
|
|
@@ -11189,7 +11357,7 @@ ${validation.errors.join("\n")}`);
|
|
|
11189
11357
|
existingHash
|
|
11190
11358
|
};
|
|
11191
11359
|
}
|
|
11192
|
-
async createResource({ name, attributes, behavior = "user-
|
|
11360
|
+
async createResource({ name, attributes, behavior = "user-managed", hooks, ...config }) {
|
|
11193
11361
|
if (this.resources[name]) {
|
|
11194
11362
|
const existingResource = this.resources[name];
|
|
11195
11363
|
Object.assign(existingResource.config, {
|
|
@@ -11199,6 +11367,7 @@ ${validation.errors.join("\n")}`);
|
|
|
11199
11367
|
if (behavior) {
|
|
11200
11368
|
existingResource.behavior = behavior;
|
|
11201
11369
|
}
|
|
11370
|
+
existingResource.versioningEnabled = this.versioningEnabled;
|
|
11202
11371
|
existingResource.updateAttributes(attributes);
|
|
11203
11372
|
if (hooks) {
|
|
11204
11373
|
for (const [event, hooksArr] of Object.entries(hooks)) {
|
|
@@ -11233,6 +11402,7 @@ ${validation.errors.join("\n")}`);
|
|
|
11233
11402
|
passphrase: this.passphrase,
|
|
11234
11403
|
cache: this.cache,
|
|
11235
11404
|
hooks,
|
|
11405
|
+
versioningEnabled: this.versioningEnabled,
|
|
11236
11406
|
...config
|
|
11237
11407
|
});
|
|
11238
11408
|
this.resources[name] = resource;
|
|
@@ -17699,12 +17869,117 @@ ${validation.errors.join("\n")}`);
|
|
|
17699
17869
|
}
|
|
17700
17870
|
|
|
17701
17871
|
class Plugin extends EventEmitter {
|
|
17872
|
+
constructor(options = {}) {
|
|
17873
|
+
super();
|
|
17874
|
+
this.name = this.constructor.name;
|
|
17875
|
+
this.options = options;
|
|
17876
|
+
this.hooks = /* @__PURE__ */ new Map();
|
|
17877
|
+
}
|
|
17702
17878
|
async setup(database) {
|
|
17879
|
+
this.database = database;
|
|
17880
|
+
this.beforeSetup();
|
|
17881
|
+
await this.onSetup();
|
|
17882
|
+
this.afterSetup();
|
|
17703
17883
|
}
|
|
17704
17884
|
async start() {
|
|
17885
|
+
this.beforeStart();
|
|
17886
|
+
await this.onStart();
|
|
17887
|
+
this.afterStart();
|
|
17705
17888
|
}
|
|
17706
17889
|
async stop() {
|
|
17890
|
+
this.beforeStop();
|
|
17891
|
+
await this.onStop();
|
|
17892
|
+
this.afterStop();
|
|
17893
|
+
}
|
|
17894
|
+
// Override these methods in subclasses
|
|
17895
|
+
async onSetup() {
|
|
17896
|
+
}
|
|
17897
|
+
async onStart() {
|
|
17707
17898
|
}
|
|
17899
|
+
async onStop() {
|
|
17900
|
+
}
|
|
17901
|
+
// Hook management methods
|
|
17902
|
+
addHook(resource, event, handler) {
|
|
17903
|
+
if (!this.hooks.has(resource)) {
|
|
17904
|
+
this.hooks.set(resource, /* @__PURE__ */ new Map());
|
|
17905
|
+
}
|
|
17906
|
+
const resourceHooks = this.hooks.get(resource);
|
|
17907
|
+
if (!resourceHooks.has(event)) {
|
|
17908
|
+
resourceHooks.set(event, []);
|
|
17909
|
+
}
|
|
17910
|
+
resourceHooks.get(event).push(handler);
|
|
17911
|
+
}
|
|
17912
|
+
removeHook(resource, event, handler) {
|
|
17913
|
+
const resourceHooks = this.hooks.get(resource);
|
|
17914
|
+
if (resourceHooks && resourceHooks.has(event)) {
|
|
17915
|
+
const handlers = resourceHooks.get(event);
|
|
17916
|
+
const index = handlers.indexOf(handler);
|
|
17917
|
+
if (index > -1) {
|
|
17918
|
+
handlers.splice(index, 1);
|
|
17919
|
+
}
|
|
17920
|
+
}
|
|
17921
|
+
}
|
|
17922
|
+
// Enhanced resource method wrapping that supports multiple plugins
|
|
17923
|
+
wrapResourceMethod(resource, methodName, wrapper) {
|
|
17924
|
+
const originalMethod = resource[methodName];
|
|
17925
|
+
if (!resource._pluginWrappers) {
|
|
17926
|
+
resource._pluginWrappers = /* @__PURE__ */ new Map();
|
|
17927
|
+
}
|
|
17928
|
+
if (!resource._pluginWrappers.has(methodName)) {
|
|
17929
|
+
resource._pluginWrappers.set(methodName, []);
|
|
17930
|
+
}
|
|
17931
|
+
resource._pluginWrappers.get(methodName).push(wrapper);
|
|
17932
|
+
if (!resource[`_wrapped_${methodName}`]) {
|
|
17933
|
+
resource[`_wrapped_${methodName}`] = originalMethod;
|
|
17934
|
+
const isJestMock = originalMethod && originalMethod._isMockFunction;
|
|
17935
|
+
resource[methodName] = async function(...args) {
|
|
17936
|
+
let result = await resource[`_wrapped_${methodName}`](...args);
|
|
17937
|
+
for (const wrapper2 of resource._pluginWrappers.get(methodName)) {
|
|
17938
|
+
result = await wrapper2.call(this, result, args, methodName);
|
|
17939
|
+
}
|
|
17940
|
+
return result;
|
|
17941
|
+
};
|
|
17942
|
+
if (isJestMock) {
|
|
17943
|
+
Object.setPrototypeOf(resource[methodName], Object.getPrototypeOf(originalMethod));
|
|
17944
|
+
Object.assign(resource[methodName], originalMethod);
|
|
17945
|
+
}
|
|
17946
|
+
}
|
|
17947
|
+
}
|
|
17948
|
+
// Partition-aware helper methods
|
|
17949
|
+
getPartitionValues(data, resource) {
|
|
17950
|
+
if (!resource.config?.partitions) return {};
|
|
17951
|
+
const partitionValues = {};
|
|
17952
|
+
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
17953
|
+
if (partitionDef.fields) {
|
|
17954
|
+
partitionValues[partitionName] = {};
|
|
17955
|
+
for (const [fieldName, rule] of Object.entries(partitionDef.fields)) {
|
|
17956
|
+
const value = this.getNestedFieldValue(data, fieldName);
|
|
17957
|
+
if (value !== null && value !== void 0) {
|
|
17958
|
+
partitionValues[partitionName][fieldName] = resource.applyPartitionRule(value, rule);
|
|
17959
|
+
}
|
|
17960
|
+
}
|
|
17961
|
+
} else {
|
|
17962
|
+
partitionValues[partitionName] = {};
|
|
17963
|
+
}
|
|
17964
|
+
}
|
|
17965
|
+
return partitionValues;
|
|
17966
|
+
}
|
|
17967
|
+
getNestedFieldValue(data, fieldPath) {
|
|
17968
|
+
if (!fieldPath.includes(".")) {
|
|
17969
|
+
return data[fieldPath] ?? null;
|
|
17970
|
+
}
|
|
17971
|
+
const keys = fieldPath.split(".");
|
|
17972
|
+
let value = data;
|
|
17973
|
+
for (const key of keys) {
|
|
17974
|
+
if (value && typeof value === "object" && key in value) {
|
|
17975
|
+
value = value[key];
|
|
17976
|
+
} else {
|
|
17977
|
+
return null;
|
|
17978
|
+
}
|
|
17979
|
+
}
|
|
17980
|
+
return value ?? null;
|
|
17981
|
+
}
|
|
17982
|
+
// Event emission methods
|
|
17708
17983
|
beforeSetup() {
|
|
17709
17984
|
this.emit("plugin.beforeSetup", /* @__PURE__ */ new Date());
|
|
17710
17985
|
}
|
|
@@ -17734,208 +18009,3066 @@ ${validation.errors.join("\n")}`);
|
|
|
17734
18009
|
}
|
|
17735
18010
|
};
|
|
17736
18011
|
|
|
17737
|
-
|
|
17738
|
-
async setup(db) {
|
|
17739
|
-
this.client = db.client;
|
|
17740
|
-
this.map = {
|
|
17741
|
-
PutObjectCommand: "put",
|
|
17742
|
-
GetObjectCommand: "get",
|
|
17743
|
-
HeadObjectCommand: "get",
|
|
17744
|
-
DeleteObjectCommand: "delete",
|
|
17745
|
-
DeleteObjectsCommand: "delete",
|
|
17746
|
-
ListObjectsV2Command: "list"
|
|
17747
|
-
};
|
|
17748
|
-
this.costs = {
|
|
17749
|
-
total: 0,
|
|
17750
|
-
prices: {
|
|
17751
|
-
put: 5e-3 / 1e3,
|
|
17752
|
-
copy: 5e-3 / 1e3,
|
|
17753
|
-
list: 5e-3 / 1e3,
|
|
17754
|
-
post: 5e-3 / 1e3,
|
|
17755
|
-
get: 4e-4 / 1e3,
|
|
17756
|
-
select: 4e-4 / 1e3,
|
|
17757
|
-
delete: 4e-4 / 1e3
|
|
17758
|
-
},
|
|
17759
|
-
requests: {
|
|
17760
|
-
total: 0,
|
|
17761
|
-
put: 0,
|
|
17762
|
-
post: 0,
|
|
17763
|
-
copy: 0,
|
|
17764
|
-
list: 0,
|
|
17765
|
-
get: 0,
|
|
17766
|
-
select: 0,
|
|
17767
|
-
delete: 0
|
|
17768
|
-
},
|
|
17769
|
-
events: {
|
|
17770
|
-
total: 0,
|
|
17771
|
-
PutObjectCommand: 0,
|
|
17772
|
-
GetObjectCommand: 0,
|
|
17773
|
-
HeadObjectCommand: 0,
|
|
17774
|
-
DeleteObjectCommand: 0,
|
|
17775
|
-
DeleteObjectsCommand: 0,
|
|
17776
|
-
ListObjectsV2Command: 0
|
|
17777
|
-
}
|
|
17778
|
-
};
|
|
17779
|
-
this.client.costs = JSON.parse(JSON.stringify(this.costs));
|
|
17780
|
-
},
|
|
17781
|
-
async start() {
|
|
17782
|
-
this.client.on("command.response", (name) => this.addRequest(name, this.map[name]));
|
|
17783
|
-
},
|
|
17784
|
-
addRequest(name, method) {
|
|
17785
|
-
this.costs.events[name]++;
|
|
17786
|
-
this.costs.events.total++;
|
|
17787
|
-
this.costs.requests.total++;
|
|
17788
|
-
this.costs.requests[method]++;
|
|
17789
|
-
this.costs.total += this.costs.prices[method];
|
|
17790
|
-
this.client.costs.events[name]++;
|
|
17791
|
-
this.client.costs.events.total++;
|
|
17792
|
-
this.client.costs.requests.total++;
|
|
17793
|
-
this.client.costs.requests[method]++;
|
|
17794
|
-
this.client.costs.total += this.client.costs.prices[method];
|
|
17795
|
-
}
|
|
17796
|
-
};
|
|
17797
|
-
|
|
17798
|
-
class CachePlugin extends Plugin {
|
|
18012
|
+
class AuditPlugin extends Plugin {
|
|
17799
18013
|
constructor(options = {}) {
|
|
17800
|
-
super();
|
|
17801
|
-
this.
|
|
18014
|
+
super(options);
|
|
18015
|
+
this.auditResource = null;
|
|
18016
|
+
this.config = {
|
|
18017
|
+
enabled: options.enabled !== false,
|
|
18018
|
+
includeData: options.includeData !== false,
|
|
18019
|
+
includePartitions: options.includePartitions !== false,
|
|
18020
|
+
maxDataSize: options.maxDataSize || 1e4,
|
|
18021
|
+
// 10KB limit
|
|
18022
|
+
...options
|
|
18023
|
+
};
|
|
17802
18024
|
}
|
|
17803
|
-
async
|
|
17804
|
-
this.
|
|
17805
|
-
|
|
17806
|
-
|
|
17807
|
-
|
|
17808
|
-
|
|
17809
|
-
|
|
17810
|
-
|
|
17811
|
-
|
|
18025
|
+
async onSetup() {
|
|
18026
|
+
if (!this.config.enabled) {
|
|
18027
|
+
this.auditResource = null;
|
|
18028
|
+
return;
|
|
18029
|
+
}
|
|
18030
|
+
try {
|
|
18031
|
+
this.auditResource = await this.database.createResource({
|
|
18032
|
+
name: "audits",
|
|
18033
|
+
attributes: {
|
|
18034
|
+
id: "string|required",
|
|
18035
|
+
resourceName: "string|required",
|
|
18036
|
+
operation: "string|required",
|
|
18037
|
+
recordId: "string|required",
|
|
18038
|
+
userId: "string|optional",
|
|
18039
|
+
timestamp: "string|required",
|
|
18040
|
+
oldData: "string|optional",
|
|
18041
|
+
newData: "string|optional",
|
|
18042
|
+
partition: "string|optional",
|
|
18043
|
+
partitionValues: "string|optional",
|
|
18044
|
+
metadata: "string|optional"
|
|
18045
|
+
}
|
|
18046
|
+
});
|
|
18047
|
+
} catch (error) {
|
|
18048
|
+
try {
|
|
18049
|
+
this.auditResource = this.database.resources.audits;
|
|
18050
|
+
} catch (innerError) {
|
|
18051
|
+
this.auditResource = null;
|
|
18052
|
+
return;
|
|
18053
|
+
}
|
|
17812
18054
|
}
|
|
18055
|
+
this.installDatabaseProxy();
|
|
18056
|
+
this.installResourceHooks();
|
|
17813
18057
|
}
|
|
17814
|
-
async
|
|
18058
|
+
async onStart() {
|
|
17815
18059
|
}
|
|
17816
|
-
async
|
|
18060
|
+
async onStop() {
|
|
17817
18061
|
}
|
|
17818
18062
|
installDatabaseProxy() {
|
|
17819
|
-
|
|
17820
|
-
|
|
18063
|
+
if (this.database._auditProxyInstalled) {
|
|
18064
|
+
return;
|
|
18065
|
+
}
|
|
18066
|
+
const installResourceHooksForResource = this.installResourceHooksForResource.bind(this);
|
|
18067
|
+
this.database._originalCreateResource = this.database.createResource;
|
|
17821
18068
|
this.database.createResource = async function(...args) {
|
|
17822
|
-
const resource = await this.
|
|
17823
|
-
|
|
18069
|
+
const resource = await this._originalCreateResource(...args);
|
|
18070
|
+
if (resource.name !== "audit_logs") {
|
|
18071
|
+
installResourceHooksForResource(resource);
|
|
18072
|
+
}
|
|
17824
18073
|
return resource;
|
|
17825
18074
|
};
|
|
17826
|
-
|
|
17827
|
-
|
|
17828
|
-
|
|
17829
|
-
|
|
17830
|
-
|
|
17831
|
-
|
|
17832
|
-
|
|
17833
|
-
|
|
17834
|
-
|
|
17835
|
-
|
|
17836
|
-
|
|
17837
|
-
|
|
17838
|
-
|
|
17839
|
-
|
|
17840
|
-
|
|
17841
|
-
|
|
17842
|
-
|
|
17843
|
-
|
|
17844
|
-
|
|
17845
|
-
|
|
17846
|
-
|
|
17847
|
-
|
|
17848
|
-
|
|
17849
|
-
|
|
17850
|
-
|
|
17851
|
-
|
|
18075
|
+
this.database._auditProxyInstalled = true;
|
|
18076
|
+
}
|
|
18077
|
+
installResourceHooks() {
|
|
18078
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18079
|
+
if (resource.name === "audits") continue;
|
|
18080
|
+
this.installResourceHooksForResource(resource);
|
|
18081
|
+
}
|
|
18082
|
+
}
|
|
18083
|
+
installResourceHooksForResource(resource) {
|
|
18084
|
+
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
18085
|
+
const [data] = args;
|
|
18086
|
+
const recordId = data.id || result.id || "auto-generated";
|
|
18087
|
+
const partitionValues = this.config.includePartitions ? this.getPartitionValues(data, resource) : null;
|
|
18088
|
+
const auditRecord = {
|
|
18089
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18090
|
+
resourceName: resource.name,
|
|
18091
|
+
operation: "insert",
|
|
18092
|
+
recordId,
|
|
18093
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18094
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18095
|
+
oldData: null,
|
|
18096
|
+
newData: this.config.includeData === false ? null : JSON.stringify(this.truncateData(data)),
|
|
18097
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18098
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18099
|
+
metadata: JSON.stringify({
|
|
18100
|
+
source: "audit-plugin",
|
|
18101
|
+
version: "2.0"
|
|
18102
|
+
})
|
|
18103
|
+
};
|
|
18104
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18105
|
+
return result;
|
|
18106
|
+
});
|
|
18107
|
+
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
18108
|
+
const [id, data] = args;
|
|
18109
|
+
let oldData = null;
|
|
18110
|
+
if (this.config.includeData) {
|
|
18111
|
+
try {
|
|
18112
|
+
oldData = await resource.get(id);
|
|
18113
|
+
} catch (error) {
|
|
18114
|
+
}
|
|
17852
18115
|
}
|
|
17853
|
-
const
|
|
17854
|
-
|
|
17855
|
-
|
|
17856
|
-
|
|
17857
|
-
|
|
17858
|
-
|
|
17859
|
-
|
|
17860
|
-
|
|
17861
|
-
|
|
17862
|
-
|
|
17863
|
-
|
|
18116
|
+
const partitionValues = this.config.includePartitions ? this.getPartitionValues(result, resource) : null;
|
|
18117
|
+
const auditRecord = {
|
|
18118
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18119
|
+
resourceName: resource.name,
|
|
18120
|
+
operation: "update",
|
|
18121
|
+
recordId: id,
|
|
18122
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18123
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18124
|
+
oldData: oldData && this.config.includeData === false ? null : oldData ? JSON.stringify(this.truncateData(oldData)) : null,
|
|
18125
|
+
newData: this.config.includeData === false ? null : JSON.stringify(this.truncateData(result)),
|
|
18126
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18127
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18128
|
+
metadata: JSON.stringify({
|
|
18129
|
+
source: "audit-plugin",
|
|
18130
|
+
version: "2.0"
|
|
18131
|
+
})
|
|
18132
|
+
};
|
|
18133
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18134
|
+
return result;
|
|
18135
|
+
});
|
|
18136
|
+
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
18137
|
+
const [id] = args;
|
|
18138
|
+
let oldData = null;
|
|
18139
|
+
if (this.config.includeData) {
|
|
18140
|
+
try {
|
|
18141
|
+
oldData = await resource.get(id);
|
|
18142
|
+
} catch (error) {
|
|
18143
|
+
}
|
|
17864
18144
|
}
|
|
17865
|
-
const
|
|
17866
|
-
|
|
17867
|
-
|
|
17868
|
-
|
|
17869
|
-
|
|
17870
|
-
|
|
17871
|
-
|
|
17872
|
-
|
|
17873
|
-
|
|
17874
|
-
|
|
17875
|
-
|
|
17876
|
-
|
|
17877
|
-
|
|
17878
|
-
|
|
18145
|
+
const partitionValues = oldData && this.config.includePartitions ? this.getPartitionValues(oldData, resource) : null;
|
|
18146
|
+
const auditRecord = {
|
|
18147
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18148
|
+
resourceName: resource.name,
|
|
18149
|
+
operation: "delete",
|
|
18150
|
+
recordId: id,
|
|
18151
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18152
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18153
|
+
oldData: oldData && this.config.includeData === false ? null : oldData ? JSON.stringify(this.truncateData(oldData)) : null,
|
|
18154
|
+
newData: null,
|
|
18155
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18156
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18157
|
+
metadata: JSON.stringify({
|
|
18158
|
+
source: "audit-plugin",
|
|
18159
|
+
version: "2.0"
|
|
18160
|
+
})
|
|
18161
|
+
};
|
|
18162
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18163
|
+
return result;
|
|
18164
|
+
});
|
|
18165
|
+
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
18166
|
+
const [ids] = args;
|
|
18167
|
+
const auditRecords = [];
|
|
18168
|
+
if (this.config.includeData) {
|
|
18169
|
+
for (const id of ids) {
|
|
18170
|
+
try {
|
|
18171
|
+
const oldData = await resource.get(id);
|
|
18172
|
+
const partitionValues = this.config.includePartitions ? this.getPartitionValues(oldData, resource) : null;
|
|
18173
|
+
auditRecords.push({
|
|
18174
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18175
|
+
resourceName: resource.name,
|
|
18176
|
+
operation: "delete",
|
|
18177
|
+
recordId: id,
|
|
18178
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18179
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18180
|
+
oldData: this.config.includeData === false ? null : JSON.stringify(this.truncateData(oldData)),
|
|
18181
|
+
newData: null,
|
|
18182
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18183
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18184
|
+
metadata: JSON.stringify({
|
|
18185
|
+
source: "audit-plugin",
|
|
18186
|
+
version: "2.0",
|
|
18187
|
+
batchOperation: true
|
|
18188
|
+
})
|
|
18189
|
+
});
|
|
18190
|
+
} catch (error) {
|
|
18191
|
+
}
|
|
18192
|
+
}
|
|
17879
18193
|
}
|
|
17880
|
-
const
|
|
17881
|
-
|
|
17882
|
-
|
|
17883
|
-
|
|
17884
|
-
|
|
17885
|
-
|
|
17886
|
-
|
|
17887
|
-
|
|
17888
|
-
|
|
17889
|
-
|
|
17890
|
-
|
|
18194
|
+
for (const auditRecord of auditRecords) {
|
|
18195
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18196
|
+
}
|
|
18197
|
+
return result;
|
|
18198
|
+
});
|
|
18199
|
+
}
|
|
18200
|
+
getPartitionValues(data, resource) {
|
|
18201
|
+
const partitions = resource.config?.partitions || {};
|
|
18202
|
+
const partitionValues = {};
|
|
18203
|
+
for (const [partitionName, partitionDef] of Object.entries(partitions)) {
|
|
18204
|
+
if (partitionDef.fields) {
|
|
18205
|
+
const partitionData = {};
|
|
18206
|
+
for (const [fieldName, fieldRule] of Object.entries(partitionDef.fields)) {
|
|
18207
|
+
const fieldValue = this.getNestedFieldValue(data, fieldName);
|
|
18208
|
+
if (fieldValue !== void 0 && fieldValue !== null) {
|
|
18209
|
+
partitionData[fieldName] = fieldValue;
|
|
18210
|
+
}
|
|
18211
|
+
}
|
|
18212
|
+
if (Object.keys(partitionData).length > 0) {
|
|
18213
|
+
partitionValues[partitionName] = partitionData;
|
|
18214
|
+
}
|
|
18215
|
+
}
|
|
18216
|
+
}
|
|
18217
|
+
return partitionValues;
|
|
18218
|
+
}
|
|
18219
|
+
getNestedFieldValue(data, fieldPath) {
|
|
18220
|
+
if (!fieldPath.includes(".")) {
|
|
18221
|
+
return data[fieldPath];
|
|
18222
|
+
}
|
|
18223
|
+
const keys = fieldPath.split(".");
|
|
18224
|
+
let currentLevel = data;
|
|
18225
|
+
for (const key of keys) {
|
|
18226
|
+
if (!currentLevel || typeof currentLevel !== "object" || !(key in currentLevel)) {
|
|
18227
|
+
return void 0;
|
|
17891
18228
|
}
|
|
17892
|
-
|
|
17893
|
-
|
|
18229
|
+
currentLevel = currentLevel[key];
|
|
18230
|
+
}
|
|
18231
|
+
return currentLevel;
|
|
18232
|
+
}
|
|
18233
|
+
getPrimaryPartition(partitionValues) {
|
|
18234
|
+
if (!partitionValues) return null;
|
|
18235
|
+
const partitionNames = Object.keys(partitionValues);
|
|
18236
|
+
return partitionNames.length > 0 ? partitionNames[0] : null;
|
|
18237
|
+
}
|
|
18238
|
+
async logAudit(auditRecord) {
|
|
18239
|
+
if (!this.auditResource) return;
|
|
18240
|
+
try {
|
|
18241
|
+
await this.auditResource.insert(auditRecord);
|
|
18242
|
+
} catch (error) {
|
|
18243
|
+
console.error("Failed to log audit record:", error);
|
|
18244
|
+
if (error && error.stack) console.error(error.stack);
|
|
18245
|
+
}
|
|
18246
|
+
}
|
|
18247
|
+
truncateData(data) {
|
|
18248
|
+
if (!data) return data;
|
|
18249
|
+
const dataStr = JSON.stringify(data);
|
|
18250
|
+
if (dataStr.length <= this.config.maxDataSize) {
|
|
17894
18251
|
return data;
|
|
18252
|
+
}
|
|
18253
|
+
return {
|
|
18254
|
+
...data,
|
|
18255
|
+
_truncated: true,
|
|
18256
|
+
_originalSize: dataStr.length,
|
|
18257
|
+
_truncatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17895
18258
|
};
|
|
17896
|
-
|
|
17897
|
-
|
|
17898
|
-
|
|
17899
|
-
|
|
17900
|
-
|
|
18259
|
+
}
|
|
18260
|
+
// Utility methods for querying audit logs
|
|
18261
|
+
async getAuditLogs(options = {}) {
|
|
18262
|
+
if (!this.auditResource) return [];
|
|
18263
|
+
try {
|
|
18264
|
+
const {
|
|
18265
|
+
resourceName,
|
|
18266
|
+
operation,
|
|
18267
|
+
recordId,
|
|
18268
|
+
userId,
|
|
18269
|
+
partition,
|
|
18270
|
+
startDate,
|
|
18271
|
+
endDate,
|
|
18272
|
+
limit = 100,
|
|
18273
|
+
offset = 0
|
|
18274
|
+
} = options;
|
|
18275
|
+
const allAudits = await this.auditResource.getAll();
|
|
18276
|
+
let filtered = allAudits.filter((audit) => {
|
|
18277
|
+
if (resourceName && audit.resourceName !== resourceName) return false;
|
|
18278
|
+
if (operation && audit.operation !== operation) return false;
|
|
18279
|
+
if (recordId && audit.recordId !== recordId) return false;
|
|
18280
|
+
if (userId && audit.userId !== userId) return false;
|
|
18281
|
+
if (partition && audit.partition !== partition) return false;
|
|
18282
|
+
if (startDate && new Date(audit.timestamp) < new Date(startDate)) return false;
|
|
18283
|
+
if (endDate && new Date(audit.timestamp) > new Date(endDate)) return false;
|
|
18284
|
+
return true;
|
|
18285
|
+
});
|
|
18286
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
18287
|
+
const deserialized = filtered.slice(offset, offset + limit).map((audit) => ({
|
|
18288
|
+
...audit,
|
|
18289
|
+
oldData: audit.oldData ? JSON.parse(audit.oldData) : null,
|
|
18290
|
+
newData: audit.newData ? JSON.parse(audit.newData) : null,
|
|
18291
|
+
partitionValues: audit.partitionValues ? JSON.parse(audit.partitionValues) : null,
|
|
18292
|
+
metadata: audit.metadata ? JSON.parse(audit.metadata) : null
|
|
18293
|
+
}));
|
|
18294
|
+
return deserialized;
|
|
18295
|
+
} catch (error) {
|
|
18296
|
+
console.error("Failed to get audit logs:", error);
|
|
18297
|
+
if (error && error.stack) console.error(error.stack);
|
|
18298
|
+
return [];
|
|
18299
|
+
}
|
|
18300
|
+
}
|
|
18301
|
+
async getRecordHistory(resourceName, recordId) {
|
|
18302
|
+
return this.getAuditLogs({
|
|
18303
|
+
resourceName,
|
|
18304
|
+
recordId,
|
|
18305
|
+
limit: 1e3
|
|
18306
|
+
});
|
|
18307
|
+
}
|
|
18308
|
+
async getPartitionHistory(resourceName, partitionName, partitionValues) {
|
|
18309
|
+
return this.getAuditLogs({
|
|
18310
|
+
resourceName,
|
|
18311
|
+
partition: partitionName,
|
|
18312
|
+
limit: 1e3
|
|
18313
|
+
});
|
|
18314
|
+
}
|
|
18315
|
+
async getAuditStats(options = {}) {
|
|
18316
|
+
const {
|
|
18317
|
+
resourceName,
|
|
18318
|
+
startDate,
|
|
18319
|
+
endDate
|
|
18320
|
+
} = options;
|
|
18321
|
+
const allAudits = await this.getAuditLogs({
|
|
18322
|
+
resourceName,
|
|
18323
|
+
startDate,
|
|
18324
|
+
endDate,
|
|
18325
|
+
limit: 1e4
|
|
18326
|
+
});
|
|
18327
|
+
const stats = {
|
|
18328
|
+
total: allAudits.length,
|
|
18329
|
+
byOperation: {},
|
|
18330
|
+
byResource: {},
|
|
18331
|
+
byPartition: {},
|
|
18332
|
+
byUser: {},
|
|
18333
|
+
timeline: {}
|
|
18334
|
+
};
|
|
18335
|
+
for (const audit of allAudits) {
|
|
18336
|
+
stats.byOperation[audit.operation] = (stats.byOperation[audit.operation] || 0) + 1;
|
|
18337
|
+
stats.byResource[audit.resourceName] = (stats.byResource[audit.resourceName] || 0) + 1;
|
|
18338
|
+
if (audit.partition) {
|
|
18339
|
+
stats.byPartition[audit.partition] = (stats.byPartition[audit.partition] || 0) + 1;
|
|
18340
|
+
}
|
|
18341
|
+
stats.byUser[audit.userId] = (stats.byUser[audit.userId] || 0) + 1;
|
|
18342
|
+
const day = audit.timestamp.split("T")[0];
|
|
18343
|
+
stats.timeline[day] = (stats.timeline[day] || 0) + 1;
|
|
18344
|
+
}
|
|
18345
|
+
return stats;
|
|
18346
|
+
}
|
|
18347
|
+
}
|
|
18348
|
+
|
|
18349
|
+
class CachePlugin extends Plugin {
|
|
18350
|
+
constructor(options = {}) {
|
|
18351
|
+
super(options);
|
|
18352
|
+
this.driver = options.driver;
|
|
18353
|
+
this.config = {
|
|
18354
|
+
enabled: options.enabled !== false,
|
|
18355
|
+
includePartitions: options.includePartitions !== false,
|
|
18356
|
+
...options
|
|
18357
|
+
};
|
|
18358
|
+
}
|
|
18359
|
+
async setup(database) {
|
|
18360
|
+
if (!this.config.enabled) {
|
|
18361
|
+
return;
|
|
18362
|
+
}
|
|
18363
|
+
await super.setup(database);
|
|
18364
|
+
}
|
|
18365
|
+
async onSetup() {
|
|
18366
|
+
if (this.config.driver) {
|
|
18367
|
+
this.driver = this.config.driver;
|
|
18368
|
+
} else if (this.config.driverType === "memory") {
|
|
18369
|
+
this.driver = new MemoryCache(this.config.memoryOptions || {});
|
|
18370
|
+
} else {
|
|
18371
|
+
this.driver = new S3Cache(this.config.s3Options || {});
|
|
18372
|
+
}
|
|
18373
|
+
this.installDatabaseProxy();
|
|
18374
|
+
this.installResourceHooks();
|
|
18375
|
+
}
|
|
18376
|
+
async onStart() {
|
|
18377
|
+
}
|
|
18378
|
+
async onStop() {
|
|
18379
|
+
}
|
|
18380
|
+
installDatabaseProxy() {
|
|
18381
|
+
if (this.database._cacheProxyInstalled) {
|
|
18382
|
+
return;
|
|
18383
|
+
}
|
|
18384
|
+
const installResourceHooks = this.installResourceHooks.bind(this);
|
|
18385
|
+
this.database._originalCreateResourceForCache = this.database.createResource;
|
|
18386
|
+
this.database.createResource = async function(...args) {
|
|
18387
|
+
const resource = await this._originalCreateResourceForCache(...args);
|
|
18388
|
+
installResourceHooks(resource);
|
|
18389
|
+
return resource;
|
|
18390
|
+
};
|
|
18391
|
+
this.database._cacheProxyInstalled = true;
|
|
18392
|
+
}
|
|
18393
|
+
installResourceHooks() {
|
|
18394
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18395
|
+
this.installResourceHooksForResource(resource);
|
|
18396
|
+
}
|
|
18397
|
+
}
|
|
18398
|
+
installResourceHooksForResource(resource) {
|
|
18399
|
+
if (!this.driver) return;
|
|
18400
|
+
resource.cache = this.driver;
|
|
18401
|
+
resource.cacheKeyFor = async (options = {}) => {
|
|
18402
|
+
const { action, params = {}, partition, partitionValues } = options;
|
|
18403
|
+
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
18404
|
+
};
|
|
18405
|
+
resource._originalCount = resource.count;
|
|
18406
|
+
resource._originalListIds = resource.listIds;
|
|
18407
|
+
resource._originalGetMany = resource.getMany;
|
|
18408
|
+
resource._originalGetAll = resource.getAll;
|
|
18409
|
+
resource._originalPage = resource.page;
|
|
18410
|
+
resource._originalList = resource.list;
|
|
18411
|
+
resource.count = async function(options = {}) {
|
|
18412
|
+
const { partition, partitionValues } = options;
|
|
18413
|
+
const key = await resource.cacheKeyFor({
|
|
18414
|
+
action: "count",
|
|
18415
|
+
partition,
|
|
18416
|
+
partitionValues
|
|
18417
|
+
});
|
|
18418
|
+
try {
|
|
18419
|
+
const cached = await resource.cache.get(key);
|
|
18420
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18421
|
+
} catch (err) {
|
|
18422
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18423
|
+
}
|
|
18424
|
+
const result = await resource._originalCount(options);
|
|
18425
|
+
await resource.cache.set(key, result);
|
|
18426
|
+
return result;
|
|
18427
|
+
};
|
|
18428
|
+
resource.listIds = async function(options = {}) {
|
|
18429
|
+
const { partition, partitionValues } = options;
|
|
18430
|
+
const key = await resource.cacheKeyFor({
|
|
18431
|
+
action: "listIds",
|
|
18432
|
+
partition,
|
|
18433
|
+
partitionValues
|
|
18434
|
+
});
|
|
18435
|
+
try {
|
|
18436
|
+
const cached = await resource.cache.get(key);
|
|
18437
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18438
|
+
} catch (err) {
|
|
18439
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18440
|
+
}
|
|
18441
|
+
const result = await resource._originalListIds(options);
|
|
18442
|
+
await resource.cache.set(key, result);
|
|
18443
|
+
return result;
|
|
18444
|
+
};
|
|
18445
|
+
resource.getMany = async function(ids) {
|
|
18446
|
+
const key = await resource.cacheKeyFor({
|
|
18447
|
+
action: "getMany",
|
|
18448
|
+
params: { ids }
|
|
18449
|
+
});
|
|
18450
|
+
try {
|
|
18451
|
+
const cached = await resource.cache.get(key);
|
|
18452
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18453
|
+
} catch (err) {
|
|
18454
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18455
|
+
}
|
|
18456
|
+
const result = await resource._originalGetMany(ids);
|
|
18457
|
+
await resource.cache.set(key, result);
|
|
18458
|
+
return result;
|
|
18459
|
+
};
|
|
18460
|
+
resource.getAll = async function() {
|
|
18461
|
+
const key = await resource.cacheKeyFor({ action: "getAll" });
|
|
18462
|
+
try {
|
|
18463
|
+
const cached = await resource.cache.get(key);
|
|
18464
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18465
|
+
} catch (err) {
|
|
18466
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18467
|
+
}
|
|
18468
|
+
const result = await resource._originalGetAll();
|
|
18469
|
+
await resource.cache.set(key, result);
|
|
18470
|
+
return result;
|
|
18471
|
+
};
|
|
18472
|
+
resource.page = async function({ offset, size, partition, partitionValues } = {}) {
|
|
18473
|
+
const key = await resource.cacheKeyFor({
|
|
18474
|
+
action: "page",
|
|
18475
|
+
params: { offset, size },
|
|
18476
|
+
partition,
|
|
18477
|
+
partitionValues
|
|
18478
|
+
});
|
|
18479
|
+
try {
|
|
18480
|
+
const cached = await resource.cache.get(key);
|
|
18481
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18482
|
+
} catch (err) {
|
|
18483
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18484
|
+
}
|
|
18485
|
+
const result = await resource._originalPage({ offset, size, partition, partitionValues });
|
|
18486
|
+
await resource.cache.set(key, result);
|
|
18487
|
+
return result;
|
|
18488
|
+
};
|
|
18489
|
+
resource.list = async function(options = {}) {
|
|
18490
|
+
const { partition, partitionValues } = options;
|
|
18491
|
+
const key = await resource.cacheKeyFor({
|
|
18492
|
+
action: "list",
|
|
18493
|
+
partition,
|
|
18494
|
+
partitionValues
|
|
18495
|
+
});
|
|
18496
|
+
try {
|
|
18497
|
+
const cached = await resource.cache.get(key);
|
|
18498
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18499
|
+
} catch (err) {
|
|
18500
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18501
|
+
}
|
|
18502
|
+
const result = await resource._originalList(options);
|
|
18503
|
+
await resource.cache.set(key, result);
|
|
18504
|
+
return result;
|
|
18505
|
+
};
|
|
18506
|
+
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
18507
|
+
const [data] = args;
|
|
18508
|
+
await this.clearCacheForResource(resource, data);
|
|
18509
|
+
return result;
|
|
18510
|
+
});
|
|
18511
|
+
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
18512
|
+
const [id, data] = args;
|
|
18513
|
+
await this.clearCacheForResource(resource, { id, ...data });
|
|
18514
|
+
return result;
|
|
18515
|
+
});
|
|
18516
|
+
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
18517
|
+
const [id] = args;
|
|
18518
|
+
let data = { id };
|
|
18519
|
+
if (typeof resource.get === "function") {
|
|
18520
|
+
try {
|
|
18521
|
+
const full = await resource.get(id);
|
|
18522
|
+
if (full) data = full;
|
|
18523
|
+
} catch {
|
|
18524
|
+
}
|
|
18525
|
+
}
|
|
18526
|
+
await this.clearCacheForResource(resource, data);
|
|
18527
|
+
return result;
|
|
18528
|
+
});
|
|
18529
|
+
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
18530
|
+
const [ids] = args;
|
|
18531
|
+
for (const id of ids) {
|
|
18532
|
+
let data = { id };
|
|
18533
|
+
if (typeof resource.get === "function") {
|
|
18534
|
+
try {
|
|
18535
|
+
const full = await resource.get(id);
|
|
18536
|
+
if (full) data = full;
|
|
18537
|
+
} catch {
|
|
18538
|
+
}
|
|
18539
|
+
}
|
|
18540
|
+
await this.clearCacheForResource(resource, data);
|
|
18541
|
+
}
|
|
18542
|
+
return result;
|
|
18543
|
+
});
|
|
18544
|
+
}
|
|
18545
|
+
async clearCacheForResource(resource, data) {
|
|
18546
|
+
if (!resource.cache) return;
|
|
18547
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
18548
|
+
await resource.cache.clear(keyPrefix);
|
|
18549
|
+
if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
|
|
18550
|
+
const partitionValues = this.getPartitionValues(data, resource);
|
|
18551
|
+
for (const [partitionName, values] of Object.entries(partitionValues)) {
|
|
18552
|
+
if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
|
|
18553
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
18554
|
+
await resource.cache.clear(partitionKeyPrefix);
|
|
18555
|
+
}
|
|
18556
|
+
}
|
|
18557
|
+
}
|
|
18558
|
+
}
|
|
18559
|
+
async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
|
|
18560
|
+
const keyParts = [
|
|
18561
|
+
`resource=${resource.name}`,
|
|
18562
|
+
`action=${action}`
|
|
18563
|
+
];
|
|
18564
|
+
if (partition && partitionValues && Object.keys(partitionValues).length > 0) {
|
|
18565
|
+
keyParts.push(`partition:${partition}`);
|
|
18566
|
+
for (const [field, value] of Object.entries(partitionValues)) {
|
|
18567
|
+
if (value !== null && value !== void 0) {
|
|
18568
|
+
keyParts.push(`${field}:${value}`);
|
|
18569
|
+
}
|
|
18570
|
+
}
|
|
18571
|
+
}
|
|
18572
|
+
if (Object.keys(params).length > 0) {
|
|
18573
|
+
const paramsHash = await this.hashParams(params);
|
|
18574
|
+
keyParts.push(paramsHash);
|
|
18575
|
+
}
|
|
18576
|
+
return join(...keyParts) + ".json.gz";
|
|
18577
|
+
}
|
|
18578
|
+
async hashParams(params) {
|
|
18579
|
+
const sortedParams = Object.keys(params).sort().map((key) => `${key}:${params[key]}`).join("|") || "empty";
|
|
18580
|
+
return await sha256(sortedParams);
|
|
18581
|
+
}
|
|
18582
|
+
// Utility methods
|
|
18583
|
+
async getCacheStats() {
|
|
18584
|
+
if (!this.driver) return null;
|
|
18585
|
+
return {
|
|
18586
|
+
size: await this.driver.size(),
|
|
18587
|
+
keys: await this.driver.keys(),
|
|
18588
|
+
driver: this.driver.constructor.name
|
|
18589
|
+
};
|
|
18590
|
+
}
|
|
18591
|
+
async clearAllCache() {
|
|
18592
|
+
if (!this.driver) return;
|
|
18593
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18594
|
+
if (resource.cache) {
|
|
18595
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
18596
|
+
await resource.cache.clear(keyPrefix);
|
|
18597
|
+
}
|
|
18598
|
+
}
|
|
18599
|
+
}
|
|
18600
|
+
async warmCache(resourceName, options = {}) {
|
|
18601
|
+
const resource = this.database.resources[resourceName];
|
|
18602
|
+
if (!resource) {
|
|
18603
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
18604
|
+
}
|
|
18605
|
+
const { includePartitions = true } = options;
|
|
18606
|
+
await resource.getAll();
|
|
18607
|
+
if (includePartitions && resource.config.partitions) {
|
|
18608
|
+
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
18609
|
+
if (partitionDef.fields) {
|
|
18610
|
+
const allRecords = await resource.getAll();
|
|
18611
|
+
const recordsArray = Array.isArray(allRecords) ? allRecords : [];
|
|
18612
|
+
const partitionValues = /* @__PURE__ */ new Set();
|
|
18613
|
+
for (const record of recordsArray.slice(0, 10)) {
|
|
18614
|
+
const values = this.getPartitionValues(record, resource);
|
|
18615
|
+
if (values[partitionName]) {
|
|
18616
|
+
partitionValues.add(JSON.stringify(values[partitionName]));
|
|
18617
|
+
}
|
|
18618
|
+
}
|
|
18619
|
+
for (const partitionValueStr of partitionValues) {
|
|
18620
|
+
const partitionValues2 = JSON.parse(partitionValueStr);
|
|
18621
|
+
await resource.list({ partition: partitionName, partitionValues: partitionValues2 });
|
|
18622
|
+
}
|
|
18623
|
+
}
|
|
18624
|
+
}
|
|
18625
|
+
}
|
|
18626
|
+
}
|
|
18627
|
+
}
|
|
18628
|
+
|
|
18629
|
+
const CostsPlugin = {
|
|
18630
|
+
async setup(db) {
|
|
18631
|
+
if (!db || !db.client) {
|
|
18632
|
+
return;
|
|
18633
|
+
}
|
|
18634
|
+
this.client = db.client;
|
|
18635
|
+
this.map = {
|
|
18636
|
+
PutObjectCommand: "put",
|
|
18637
|
+
GetObjectCommand: "get",
|
|
18638
|
+
HeadObjectCommand: "head",
|
|
18639
|
+
DeleteObjectCommand: "delete",
|
|
18640
|
+
DeleteObjectsCommand: "delete",
|
|
18641
|
+
ListObjectsV2Command: "list"
|
|
18642
|
+
};
|
|
18643
|
+
this.costs = {
|
|
18644
|
+
total: 0,
|
|
18645
|
+
prices: {
|
|
18646
|
+
put: 5e-3 / 1e3,
|
|
18647
|
+
copy: 5e-3 / 1e3,
|
|
18648
|
+
list: 5e-3 / 1e3,
|
|
18649
|
+
post: 5e-3 / 1e3,
|
|
18650
|
+
get: 4e-4 / 1e3,
|
|
18651
|
+
select: 4e-4 / 1e3,
|
|
18652
|
+
delete: 4e-4 / 1e3,
|
|
18653
|
+
head: 4e-4 / 1e3
|
|
18654
|
+
},
|
|
18655
|
+
requests: {
|
|
18656
|
+
total: 0,
|
|
18657
|
+
put: 0,
|
|
18658
|
+
post: 0,
|
|
18659
|
+
copy: 0,
|
|
18660
|
+
list: 0,
|
|
18661
|
+
get: 0,
|
|
18662
|
+
select: 0,
|
|
18663
|
+
delete: 0,
|
|
18664
|
+
head: 0
|
|
18665
|
+
},
|
|
18666
|
+
events: {
|
|
18667
|
+
total: 0,
|
|
18668
|
+
PutObjectCommand: 0,
|
|
18669
|
+
GetObjectCommand: 0,
|
|
18670
|
+
HeadObjectCommand: 0,
|
|
18671
|
+
DeleteObjectCommand: 0,
|
|
18672
|
+
DeleteObjectsCommand: 0,
|
|
18673
|
+
ListObjectsV2Command: 0
|
|
18674
|
+
}
|
|
18675
|
+
};
|
|
18676
|
+
this.client.costs = JSON.parse(JSON.stringify(this.costs));
|
|
18677
|
+
},
|
|
18678
|
+
async start() {
|
|
18679
|
+
if (this.client) {
|
|
18680
|
+
this.client.on("command.response", (name) => this.addRequest(name, this.map[name]));
|
|
18681
|
+
this.client.on("command.error", (name) => this.addRequest(name, this.map[name]));
|
|
18682
|
+
}
|
|
18683
|
+
},
|
|
18684
|
+
addRequest(name, method) {
|
|
18685
|
+
if (!method) return;
|
|
18686
|
+
this.costs.events[name]++;
|
|
18687
|
+
this.costs.events.total++;
|
|
18688
|
+
this.costs.requests.total++;
|
|
18689
|
+
this.costs.requests[method]++;
|
|
18690
|
+
this.costs.total += this.costs.prices[method];
|
|
18691
|
+
if (this.client && this.client.costs) {
|
|
18692
|
+
this.client.costs.events[name]++;
|
|
18693
|
+
this.client.costs.events.total++;
|
|
18694
|
+
this.client.costs.requests.total++;
|
|
18695
|
+
this.client.costs.requests[method]++;
|
|
18696
|
+
this.client.costs.total += this.client.costs.prices[method];
|
|
18697
|
+
}
|
|
18698
|
+
}
|
|
18699
|
+
};
|
|
18700
|
+
|
|
18701
|
+
class FullTextPlugin extends Plugin {
|
|
18702
|
+
constructor(options = {}) {
|
|
18703
|
+
super();
|
|
18704
|
+
this.indexResource = null;
|
|
18705
|
+
this.config = {
|
|
18706
|
+
enabled: options.enabled !== false,
|
|
18707
|
+
minWordLength: options.minWordLength || 3,
|
|
18708
|
+
maxResults: options.maxResults || 100,
|
|
18709
|
+
...options
|
|
18710
|
+
};
|
|
18711
|
+
this.indexes = /* @__PURE__ */ new Map();
|
|
18712
|
+
}
|
|
18713
|
+
async setup(database) {
|
|
18714
|
+
this.database = database;
|
|
18715
|
+
if (!this.config.enabled) return;
|
|
18716
|
+
try {
|
|
18717
|
+
this.indexResource = await database.createResource({
|
|
18718
|
+
name: "fulltext_indexes",
|
|
18719
|
+
attributes: {
|
|
18720
|
+
id: "string|required",
|
|
18721
|
+
resourceName: "string|required",
|
|
18722
|
+
fieldName: "string|required",
|
|
18723
|
+
word: "string|required",
|
|
18724
|
+
recordIds: "json|required",
|
|
18725
|
+
// Array of record IDs containing this word
|
|
18726
|
+
count: "number|required",
|
|
18727
|
+
lastUpdated: "string|required"
|
|
18728
|
+
}
|
|
18729
|
+
});
|
|
18730
|
+
} catch (error) {
|
|
18731
|
+
this.indexResource = database.resources.fulltext_indexes;
|
|
18732
|
+
}
|
|
18733
|
+
await this.loadIndexes();
|
|
18734
|
+
this.installIndexingHooks();
|
|
18735
|
+
}
|
|
18736
|
+
async start() {
|
|
18737
|
+
}
|
|
18738
|
+
async stop() {
|
|
18739
|
+
await this.saveIndexes();
|
|
18740
|
+
}
|
|
18741
|
+
async loadIndexes() {
|
|
18742
|
+
if (!this.indexResource) return;
|
|
18743
|
+
try {
|
|
18744
|
+
const allIndexes = await this.indexResource.getAll();
|
|
18745
|
+
for (const indexRecord of allIndexes) {
|
|
18746
|
+
const key = `${indexRecord.resourceName}:${indexRecord.fieldName}:${indexRecord.word}`;
|
|
18747
|
+
this.indexes.set(key, {
|
|
18748
|
+
recordIds: indexRecord.recordIds || [],
|
|
18749
|
+
count: indexRecord.count || 0
|
|
18750
|
+
});
|
|
18751
|
+
}
|
|
18752
|
+
} catch (error) {
|
|
18753
|
+
console.warn("Failed to load existing indexes:", error.message);
|
|
18754
|
+
}
|
|
18755
|
+
}
|
|
18756
|
+
async saveIndexes() {
|
|
18757
|
+
if (!this.indexResource) return;
|
|
18758
|
+
try {
|
|
18759
|
+
const existingIndexes = await this.indexResource.getAll();
|
|
18760
|
+
for (const index of existingIndexes) {
|
|
18761
|
+
await this.indexResource.delete(index.id);
|
|
18762
|
+
}
|
|
18763
|
+
for (const [key, data] of this.indexes.entries()) {
|
|
18764
|
+
const [resourceName, fieldName, word] = key.split(":");
|
|
18765
|
+
await this.indexResource.insert({
|
|
18766
|
+
id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18767
|
+
resourceName,
|
|
18768
|
+
fieldName,
|
|
18769
|
+
word,
|
|
18770
|
+
recordIds: data.recordIds,
|
|
18771
|
+
count: data.count,
|
|
18772
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
18773
|
+
});
|
|
18774
|
+
}
|
|
18775
|
+
} catch (error) {
|
|
18776
|
+
console.error("Failed to save indexes:", error);
|
|
18777
|
+
}
|
|
18778
|
+
}
|
|
18779
|
+
installIndexingHooks() {
|
|
18780
|
+
if (!this.database.plugins) {
|
|
18781
|
+
this.database.plugins = {};
|
|
18782
|
+
}
|
|
18783
|
+
this.database.plugins.fulltext = this;
|
|
18784
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18785
|
+
if (resource.name === "fulltext_indexes") continue;
|
|
18786
|
+
this.installResourceHooks(resource);
|
|
18787
|
+
}
|
|
18788
|
+
if (!this.database._fulltextProxyInstalled) {
|
|
18789
|
+
this.database._previousCreateResourceForFullText = this.database.createResource;
|
|
18790
|
+
this.database.createResource = async function(...args) {
|
|
18791
|
+
const resource = await this._previousCreateResourceForFullText(...args);
|
|
18792
|
+
if (this.plugins?.fulltext && resource.name !== "fulltext_indexes") {
|
|
18793
|
+
this.plugins.fulltext.installResourceHooks(resource);
|
|
18794
|
+
}
|
|
18795
|
+
return resource;
|
|
18796
|
+
};
|
|
18797
|
+
this.database._fulltextProxyInstalled = true;
|
|
18798
|
+
}
|
|
18799
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18800
|
+
if (resource.name !== "fulltext_indexes") {
|
|
18801
|
+
this.installResourceHooks(resource);
|
|
18802
|
+
}
|
|
18803
|
+
}
|
|
18804
|
+
}
|
|
18805
|
+
installResourceHooks(resource) {
|
|
18806
|
+
resource._insert = resource.insert;
|
|
18807
|
+
resource._update = resource.update;
|
|
18808
|
+
resource._delete = resource.delete;
|
|
18809
|
+
resource._deleteMany = resource.deleteMany;
|
|
18810
|
+
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
18811
|
+
const [data] = args;
|
|
18812
|
+
this.indexRecord(resource.name, result.id, data).catch(console.error);
|
|
18813
|
+
return result;
|
|
18814
|
+
});
|
|
18815
|
+
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
18816
|
+
const [id, data] = args;
|
|
18817
|
+
this.removeRecordFromIndex(resource.name, id).catch(console.error);
|
|
18818
|
+
this.indexRecord(resource.name, id, result).catch(console.error);
|
|
18819
|
+
return result;
|
|
18820
|
+
});
|
|
18821
|
+
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
18822
|
+
const [id] = args;
|
|
18823
|
+
this.removeRecordFromIndex(resource.name, id).catch(console.error);
|
|
18824
|
+
return result;
|
|
18825
|
+
});
|
|
18826
|
+
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
18827
|
+
const [ids] = args;
|
|
18828
|
+
for (const id of ids) {
|
|
18829
|
+
this.removeRecordFromIndex(resource.name, id).catch(console.error);
|
|
18830
|
+
}
|
|
18831
|
+
return result;
|
|
18832
|
+
});
|
|
18833
|
+
}
|
|
18834
|
+
async indexRecord(resourceName, recordId, data) {
|
|
18835
|
+
const indexedFields = this.getIndexedFields(resourceName);
|
|
18836
|
+
if (!indexedFields || indexedFields.length === 0) return;
|
|
18837
|
+
for (const fieldName of indexedFields) {
|
|
18838
|
+
const fieldValue = this.getFieldValue(data, fieldName);
|
|
18839
|
+
if (!fieldValue) continue;
|
|
18840
|
+
const words = this.tokenize(fieldValue);
|
|
18841
|
+
for (const word of words) {
|
|
18842
|
+
if (word.length < this.config.minWordLength) continue;
|
|
18843
|
+
const key = `${resourceName}:${fieldName}:${word.toLowerCase()}`;
|
|
18844
|
+
const existing = this.indexes.get(key) || { recordIds: [], count: 0 };
|
|
18845
|
+
if (!existing.recordIds.includes(recordId)) {
|
|
18846
|
+
existing.recordIds.push(recordId);
|
|
18847
|
+
existing.count = existing.recordIds.length;
|
|
18848
|
+
}
|
|
18849
|
+
this.indexes.set(key, existing);
|
|
18850
|
+
}
|
|
18851
|
+
}
|
|
18852
|
+
}
|
|
18853
|
+
async removeRecordFromIndex(resourceName, recordId) {
|
|
18854
|
+
for (const [key, data] of this.indexes.entries()) {
|
|
18855
|
+
if (key.startsWith(`${resourceName}:`)) {
|
|
18856
|
+
const index = data.recordIds.indexOf(recordId);
|
|
18857
|
+
if (index > -1) {
|
|
18858
|
+
data.recordIds.splice(index, 1);
|
|
18859
|
+
data.count = data.recordIds.length;
|
|
18860
|
+
if (data.recordIds.length === 0) {
|
|
18861
|
+
this.indexes.delete(key);
|
|
18862
|
+
} else {
|
|
18863
|
+
this.indexes.set(key, data);
|
|
18864
|
+
}
|
|
18865
|
+
}
|
|
18866
|
+
}
|
|
18867
|
+
}
|
|
18868
|
+
}
|
|
18869
|
+
getFieldValue(data, fieldPath) {
|
|
18870
|
+
if (!fieldPath.includes(".")) {
|
|
18871
|
+
return data[fieldPath];
|
|
18872
|
+
}
|
|
18873
|
+
const keys = fieldPath.split(".");
|
|
18874
|
+
let value = data;
|
|
18875
|
+
for (const key of keys) {
|
|
18876
|
+
if (value && typeof value === "object" && key in value) {
|
|
18877
|
+
value = value[key];
|
|
18878
|
+
} else {
|
|
18879
|
+
return null;
|
|
18880
|
+
}
|
|
18881
|
+
}
|
|
18882
|
+
return value;
|
|
18883
|
+
}
|
|
18884
|
+
tokenize(text) {
|
|
18885
|
+
if (!text) return [];
|
|
18886
|
+
const str = String(text).toLowerCase();
|
|
18887
|
+
return str.replace(/[^\w\s\u00C0-\u017F]/g, " ").split(/\s+/).filter((word) => word.length > 0);
|
|
18888
|
+
}
|
|
18889
|
+
getIndexedFields(resourceName) {
|
|
18890
|
+
if (this.config.fields) {
|
|
18891
|
+
return this.config.fields;
|
|
18892
|
+
}
|
|
18893
|
+
const fieldMappings = {
|
|
18894
|
+
users: ["name", "email"],
|
|
18895
|
+
products: ["name", "description"],
|
|
18896
|
+
articles: ["title", "content"]
|
|
18897
|
+
// Add more mappings as needed
|
|
18898
|
+
};
|
|
18899
|
+
return fieldMappings[resourceName] || [];
|
|
18900
|
+
}
|
|
18901
|
+
// Main search method
|
|
18902
|
+
async search(resourceName, query, options = {}) {
|
|
18903
|
+
const {
|
|
18904
|
+
fields = null,
|
|
18905
|
+
// Specific fields to search in
|
|
18906
|
+
limit = this.config.maxResults,
|
|
18907
|
+
offset = 0,
|
|
18908
|
+
exactMatch = false
|
|
18909
|
+
} = options;
|
|
18910
|
+
if (!query || query.trim().length === 0) {
|
|
18911
|
+
return [];
|
|
18912
|
+
}
|
|
18913
|
+
const searchWords = this.tokenize(query);
|
|
18914
|
+
const results = /* @__PURE__ */ new Map();
|
|
18915
|
+
const searchFields = fields || this.getIndexedFields(resourceName);
|
|
18916
|
+
if (searchFields.length === 0) {
|
|
18917
|
+
return [];
|
|
18918
|
+
}
|
|
18919
|
+
for (const word of searchWords) {
|
|
18920
|
+
if (word.length < this.config.minWordLength) continue;
|
|
18921
|
+
for (const fieldName of searchFields) {
|
|
18922
|
+
if (exactMatch) {
|
|
18923
|
+
const key = `${resourceName}:${fieldName}:${word.toLowerCase()}`;
|
|
18924
|
+
const indexData = this.indexes.get(key);
|
|
18925
|
+
if (indexData) {
|
|
18926
|
+
for (const recordId of indexData.recordIds) {
|
|
18927
|
+
const currentScore = results.get(recordId) || 0;
|
|
18928
|
+
results.set(recordId, currentScore + 1);
|
|
18929
|
+
}
|
|
18930
|
+
}
|
|
18931
|
+
} else {
|
|
18932
|
+
for (const [key, indexData] of this.indexes.entries()) {
|
|
18933
|
+
if (key.startsWith(`${resourceName}:${fieldName}:${word.toLowerCase()}`)) {
|
|
18934
|
+
for (const recordId of indexData.recordIds) {
|
|
18935
|
+
const currentScore = results.get(recordId) || 0;
|
|
18936
|
+
results.set(recordId, currentScore + 1);
|
|
18937
|
+
}
|
|
18938
|
+
}
|
|
18939
|
+
}
|
|
18940
|
+
}
|
|
18941
|
+
}
|
|
18942
|
+
}
|
|
18943
|
+
const sortedResults = Array.from(results.entries()).map(([recordId, score]) => ({ recordId, score })).sort((a, b) => b.score - a.score).slice(offset, offset + limit);
|
|
18944
|
+
return sortedResults;
|
|
18945
|
+
}
|
|
18946
|
+
// Search and return full records
|
|
18947
|
+
async searchRecords(resourceName, query, options = {}) {
|
|
18948
|
+
const searchResults = await this.search(resourceName, query, options);
|
|
18949
|
+
if (searchResults.length === 0) {
|
|
18950
|
+
return [];
|
|
18951
|
+
}
|
|
18952
|
+
const resource = this.database.resources[resourceName];
|
|
18953
|
+
if (!resource) {
|
|
18954
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
18955
|
+
}
|
|
18956
|
+
const recordIds = searchResults.map((result) => result.recordId);
|
|
18957
|
+
const records = await resource.getMany(recordIds);
|
|
18958
|
+
return records.map((record) => {
|
|
18959
|
+
const searchResult = searchResults.find((sr) => sr.recordId === record.id);
|
|
18960
|
+
return {
|
|
18961
|
+
...record,
|
|
18962
|
+
_searchScore: searchResult ? searchResult.score : 0
|
|
18963
|
+
};
|
|
18964
|
+
}).sort((a, b) => b._searchScore - a._searchScore);
|
|
18965
|
+
}
|
|
18966
|
+
// Utility methods
|
|
18967
|
+
async rebuildIndex(resourceName) {
|
|
18968
|
+
const resource = this.database.resources[resourceName];
|
|
18969
|
+
if (!resource) {
|
|
18970
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
18971
|
+
}
|
|
18972
|
+
for (const [key] of this.indexes.entries()) {
|
|
18973
|
+
if (key.startsWith(`${resourceName}:`)) {
|
|
18974
|
+
this.indexes.delete(key);
|
|
18975
|
+
}
|
|
18976
|
+
}
|
|
18977
|
+
const allRecords = await resource.getAll();
|
|
18978
|
+
const batchSize = 100;
|
|
18979
|
+
for (let i = 0; i < allRecords.length; i += batchSize) {
|
|
18980
|
+
const batch = allRecords.slice(i, i + batchSize);
|
|
18981
|
+
for (const record of batch) {
|
|
18982
|
+
await this.indexRecord(resourceName, record.id, record);
|
|
18983
|
+
}
|
|
18984
|
+
}
|
|
18985
|
+
await this.saveIndexes();
|
|
18986
|
+
}
|
|
18987
|
+
async getIndexStats() {
|
|
18988
|
+
const stats = {
|
|
18989
|
+
totalIndexes: this.indexes.size,
|
|
18990
|
+
resources: {},
|
|
18991
|
+
totalWords: 0
|
|
18992
|
+
};
|
|
18993
|
+
for (const [key, data] of this.indexes.entries()) {
|
|
18994
|
+
const [resourceName, fieldName] = key.split(":");
|
|
18995
|
+
if (!stats.resources[resourceName]) {
|
|
18996
|
+
stats.resources[resourceName] = {
|
|
18997
|
+
fields: {},
|
|
18998
|
+
totalRecords: /* @__PURE__ */ new Set(),
|
|
18999
|
+
totalWords: 0
|
|
19000
|
+
};
|
|
19001
|
+
}
|
|
19002
|
+
if (!stats.resources[resourceName].fields[fieldName]) {
|
|
19003
|
+
stats.resources[resourceName].fields[fieldName] = {
|
|
19004
|
+
words: 0,
|
|
19005
|
+
totalOccurrences: 0
|
|
19006
|
+
};
|
|
19007
|
+
}
|
|
19008
|
+
stats.resources[resourceName].fields[fieldName].words++;
|
|
19009
|
+
stats.resources[resourceName].fields[fieldName].totalOccurrences += data.count;
|
|
19010
|
+
stats.resources[resourceName].totalWords++;
|
|
19011
|
+
for (const recordId of data.recordIds) {
|
|
19012
|
+
stats.resources[resourceName].totalRecords.add(recordId);
|
|
19013
|
+
}
|
|
19014
|
+
stats.totalWords++;
|
|
19015
|
+
}
|
|
19016
|
+
for (const resourceName in stats.resources) {
|
|
19017
|
+
stats.resources[resourceName].totalRecords = stats.resources[resourceName].totalRecords.size;
|
|
19018
|
+
}
|
|
19019
|
+
return stats;
|
|
19020
|
+
}
|
|
19021
|
+
async rebuildAllIndexes({ timeout } = {}) {
|
|
19022
|
+
if (timeout) {
|
|
19023
|
+
return Promise.race([
|
|
19024
|
+
this._rebuildAllIndexesInternal(),
|
|
19025
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout")), timeout))
|
|
19026
|
+
]);
|
|
19027
|
+
}
|
|
19028
|
+
return this._rebuildAllIndexesInternal();
|
|
19029
|
+
}
|
|
19030
|
+
async _rebuildAllIndexesInternal() {
|
|
19031
|
+
const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "fulltext_indexes");
|
|
19032
|
+
for (const resourceName of resourceNames) {
|
|
19033
|
+
try {
|
|
19034
|
+
await this.rebuildIndex(resourceName);
|
|
19035
|
+
} catch (error) {
|
|
19036
|
+
console.warn(`Failed to rebuild index for resource ${resourceName}:`, error.message);
|
|
19037
|
+
}
|
|
19038
|
+
}
|
|
19039
|
+
}
|
|
19040
|
+
async clearIndex(resourceName) {
|
|
19041
|
+
for (const [key] of this.indexes.entries()) {
|
|
19042
|
+
if (key.startsWith(`${resourceName}:`)) {
|
|
19043
|
+
this.indexes.delete(key);
|
|
19044
|
+
}
|
|
19045
|
+
}
|
|
19046
|
+
await this.saveIndexes();
|
|
19047
|
+
}
|
|
19048
|
+
async clearAllIndexes() {
|
|
19049
|
+
this.indexes.clear();
|
|
19050
|
+
await this.saveIndexes();
|
|
19051
|
+
}
|
|
19052
|
+
}
|
|
19053
|
+
|
|
19054
|
+
class MetricsPlugin extends Plugin {
|
|
19055
|
+
constructor(options = {}) {
|
|
19056
|
+
super();
|
|
19057
|
+
this.config = {
|
|
19058
|
+
enabled: options.enabled !== false,
|
|
19059
|
+
collectPerformance: options.collectPerformance !== false,
|
|
19060
|
+
collectErrors: options.collectErrors !== false,
|
|
19061
|
+
collectUsage: options.collectUsage !== false,
|
|
19062
|
+
retentionDays: options.retentionDays || 30,
|
|
19063
|
+
flushInterval: options.flushInterval || 6e4,
|
|
19064
|
+
// 1 minute
|
|
19065
|
+
...options
|
|
19066
|
+
};
|
|
19067
|
+
this.metrics = {
|
|
19068
|
+
operations: {
|
|
19069
|
+
insert: { count: 0, totalTime: 0, errors: 0 },
|
|
19070
|
+
update: { count: 0, totalTime: 0, errors: 0 },
|
|
19071
|
+
delete: { count: 0, totalTime: 0, errors: 0 },
|
|
19072
|
+
get: { count: 0, totalTime: 0, errors: 0 },
|
|
19073
|
+
list: { count: 0, totalTime: 0, errors: 0 },
|
|
19074
|
+
count: { count: 0, totalTime: 0, errors: 0 }
|
|
19075
|
+
},
|
|
19076
|
+
resources: {},
|
|
19077
|
+
errors: [],
|
|
19078
|
+
performance: [],
|
|
19079
|
+
startTime: (/* @__PURE__ */ new Date()).toISOString()
|
|
19080
|
+
};
|
|
19081
|
+
this.flushTimer = null;
|
|
19082
|
+
}
|
|
19083
|
+
async setup(database) {
|
|
19084
|
+
this.database = database;
|
|
19085
|
+
if (!this.config.enabled || process.env.NODE_ENV === "test") return;
|
|
19086
|
+
try {
|
|
19087
|
+
this.metricsResource = await database.createResource({
|
|
19088
|
+
name: "metrics",
|
|
19089
|
+
attributes: {
|
|
19090
|
+
id: "string|required",
|
|
19091
|
+
type: "string|required",
|
|
19092
|
+
// 'operation', 'error', 'performance'
|
|
19093
|
+
resourceName: "string",
|
|
19094
|
+
operation: "string",
|
|
19095
|
+
count: "number|required",
|
|
19096
|
+
totalTime: "number|required",
|
|
19097
|
+
errors: "number|required",
|
|
19098
|
+
avgTime: "number|required",
|
|
19099
|
+
timestamp: "string|required",
|
|
19100
|
+
metadata: "json"
|
|
19101
|
+
}
|
|
19102
|
+
});
|
|
19103
|
+
this.errorsResource = await database.createResource({
|
|
19104
|
+
name: "error_logs",
|
|
19105
|
+
attributes: {
|
|
19106
|
+
id: "string|required",
|
|
19107
|
+
resourceName: "string|required",
|
|
19108
|
+
operation: "string|required",
|
|
19109
|
+
error: "string|required",
|
|
19110
|
+
timestamp: "string|required",
|
|
19111
|
+
metadata: "json"
|
|
19112
|
+
}
|
|
19113
|
+
});
|
|
19114
|
+
this.performanceResource = await database.createResource({
|
|
19115
|
+
name: "performance_logs",
|
|
19116
|
+
attributes: {
|
|
19117
|
+
id: "string|required",
|
|
19118
|
+
resourceName: "string|required",
|
|
19119
|
+
operation: "string|required",
|
|
19120
|
+
duration: "number|required",
|
|
19121
|
+
timestamp: "string|required",
|
|
19122
|
+
metadata: "json"
|
|
19123
|
+
}
|
|
19124
|
+
});
|
|
19125
|
+
} catch (error) {
|
|
19126
|
+
this.metricsResource = database.resources.metrics;
|
|
19127
|
+
this.errorsResource = database.resources.error_logs;
|
|
19128
|
+
this.performanceResource = database.resources.performance_logs;
|
|
19129
|
+
}
|
|
19130
|
+
this.installMetricsHooks();
|
|
19131
|
+
if (process.env.NODE_ENV !== "test") {
|
|
19132
|
+
this.startFlushTimer();
|
|
19133
|
+
}
|
|
19134
|
+
}
|
|
19135
|
+
async start() {
|
|
19136
|
+
}
|
|
19137
|
+
async stop() {
|
|
19138
|
+
if (this.flushTimer) {
|
|
19139
|
+
clearInterval(this.flushTimer);
|
|
19140
|
+
this.flushTimer = null;
|
|
19141
|
+
}
|
|
19142
|
+
if (process.env.NODE_ENV !== "test") {
|
|
19143
|
+
await this.flushMetrics();
|
|
19144
|
+
}
|
|
19145
|
+
}
|
|
19146
|
+
installMetricsHooks() {
|
|
19147
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
19148
|
+
if (["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
|
|
19149
|
+
continue;
|
|
19150
|
+
}
|
|
19151
|
+
this.installResourceHooks(resource);
|
|
19152
|
+
}
|
|
19153
|
+
this.database._createResource = this.database.createResource;
|
|
19154
|
+
this.database.createResource = async function(...args) {
|
|
19155
|
+
const resource = await this._createResource(...args);
|
|
19156
|
+
if (this.plugins?.metrics && !["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
|
|
19157
|
+
this.plugins.metrics.installResourceHooks(resource);
|
|
19158
|
+
}
|
|
19159
|
+
return resource;
|
|
19160
|
+
};
|
|
19161
|
+
}
|
|
19162
|
+
installResourceHooks(resource) {
|
|
19163
|
+
resource._insert = resource.insert;
|
|
19164
|
+
resource._update = resource.update;
|
|
19165
|
+
resource._delete = resource.delete;
|
|
19166
|
+
resource._deleteMany = resource.deleteMany;
|
|
19167
|
+
resource._get = resource.get;
|
|
19168
|
+
resource._getMany = resource.getMany;
|
|
19169
|
+
resource._getAll = resource.getAll;
|
|
19170
|
+
resource._list = resource.list;
|
|
19171
|
+
resource._listIds = resource.listIds;
|
|
19172
|
+
resource._count = resource.count;
|
|
19173
|
+
resource._page = resource.page;
|
|
19174
|
+
resource.insert = async function(...args) {
|
|
19175
|
+
const startTime = Date.now();
|
|
19176
|
+
try {
|
|
19177
|
+
const result = await resource._insert(...args);
|
|
19178
|
+
this.recordOperation(resource.name, "insert", Date.now() - startTime, false);
|
|
19179
|
+
return result;
|
|
19180
|
+
} catch (error) {
|
|
19181
|
+
this.recordOperation(resource.name, "insert", Date.now() - startTime, true);
|
|
19182
|
+
this.recordError(resource.name, "insert", error);
|
|
19183
|
+
throw error;
|
|
19184
|
+
}
|
|
19185
|
+
}.bind(this);
|
|
19186
|
+
resource.update = async function(...args) {
|
|
19187
|
+
const startTime = Date.now();
|
|
19188
|
+
try {
|
|
19189
|
+
const result = await resource._update(...args);
|
|
19190
|
+
this.recordOperation(resource.name, "update", Date.now() - startTime, false);
|
|
19191
|
+
return result;
|
|
19192
|
+
} catch (error) {
|
|
19193
|
+
this.recordOperation(resource.name, "update", Date.now() - startTime, true);
|
|
19194
|
+
this.recordError(resource.name, "update", error);
|
|
19195
|
+
throw error;
|
|
19196
|
+
}
|
|
19197
|
+
}.bind(this);
|
|
19198
|
+
resource.delete = async function(...args) {
|
|
19199
|
+
const startTime = Date.now();
|
|
19200
|
+
try {
|
|
19201
|
+
const result = await resource._delete(...args);
|
|
19202
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, false);
|
|
19203
|
+
return result;
|
|
19204
|
+
} catch (error) {
|
|
19205
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, true);
|
|
19206
|
+
this.recordError(resource.name, "delete", error);
|
|
19207
|
+
throw error;
|
|
19208
|
+
}
|
|
19209
|
+
}.bind(this);
|
|
19210
|
+
resource.deleteMany = async function(...args) {
|
|
19211
|
+
const startTime = Date.now();
|
|
19212
|
+
try {
|
|
19213
|
+
const result = await resource._deleteMany(...args);
|
|
19214
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, false);
|
|
19215
|
+
return result;
|
|
19216
|
+
} catch (error) {
|
|
19217
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, true);
|
|
19218
|
+
this.recordError(resource.name, "delete", error);
|
|
19219
|
+
throw error;
|
|
19220
|
+
}
|
|
19221
|
+
}.bind(this);
|
|
19222
|
+
resource.get = async function(...args) {
|
|
19223
|
+
const startTime = Date.now();
|
|
19224
|
+
try {
|
|
19225
|
+
const result = await resource._get(...args);
|
|
19226
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, false);
|
|
19227
|
+
return result;
|
|
19228
|
+
} catch (error) {
|
|
19229
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, true);
|
|
19230
|
+
this.recordError(resource.name, "get", error);
|
|
19231
|
+
throw error;
|
|
19232
|
+
}
|
|
19233
|
+
}.bind(this);
|
|
19234
|
+
resource.getMany = async function(...args) {
|
|
19235
|
+
const startTime = Date.now();
|
|
19236
|
+
try {
|
|
19237
|
+
const result = await resource._getMany(...args);
|
|
19238
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, false);
|
|
19239
|
+
return result;
|
|
19240
|
+
} catch (error) {
|
|
19241
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, true);
|
|
19242
|
+
this.recordError(resource.name, "get", error);
|
|
19243
|
+
throw error;
|
|
19244
|
+
}
|
|
19245
|
+
}.bind(this);
|
|
19246
|
+
resource.getAll = async function(...args) {
|
|
19247
|
+
const startTime = Date.now();
|
|
19248
|
+
try {
|
|
19249
|
+
const result = await resource._getAll(...args);
|
|
19250
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19251
|
+
return result;
|
|
19252
|
+
} catch (error) {
|
|
19253
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19254
|
+
this.recordError(resource.name, "list", error);
|
|
19255
|
+
throw error;
|
|
19256
|
+
}
|
|
19257
|
+
}.bind(this);
|
|
19258
|
+
resource.list = async function(...args) {
|
|
19259
|
+
const startTime = Date.now();
|
|
19260
|
+
try {
|
|
19261
|
+
const result = await resource._list(...args);
|
|
19262
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19263
|
+
return result;
|
|
19264
|
+
} catch (error) {
|
|
19265
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19266
|
+
this.recordError(resource.name, "list", error);
|
|
19267
|
+
throw error;
|
|
19268
|
+
}
|
|
19269
|
+
}.bind(this);
|
|
19270
|
+
resource.listIds = async function(...args) {
|
|
19271
|
+
const startTime = Date.now();
|
|
19272
|
+
try {
|
|
19273
|
+
const result = await resource._listIds(...args);
|
|
19274
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19275
|
+
return result;
|
|
19276
|
+
} catch (error) {
|
|
19277
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19278
|
+
this.recordError(resource.name, "list", error);
|
|
19279
|
+
throw error;
|
|
19280
|
+
}
|
|
19281
|
+
}.bind(this);
|
|
19282
|
+
resource.count = async function(...args) {
|
|
19283
|
+
const startTime = Date.now();
|
|
19284
|
+
try {
|
|
19285
|
+
const result = await resource._count(...args);
|
|
19286
|
+
this.recordOperation(resource.name, "count", Date.now() - startTime, false);
|
|
19287
|
+
return result;
|
|
19288
|
+
} catch (error) {
|
|
19289
|
+
this.recordOperation(resource.name, "count", Date.now() - startTime, true);
|
|
19290
|
+
this.recordError(resource.name, "count", error);
|
|
19291
|
+
throw error;
|
|
19292
|
+
}
|
|
19293
|
+
}.bind(this);
|
|
19294
|
+
resource.page = async function(...args) {
|
|
19295
|
+
const startTime = Date.now();
|
|
19296
|
+
try {
|
|
19297
|
+
const result = await resource._page(...args);
|
|
19298
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19299
|
+
return result;
|
|
19300
|
+
} catch (error) {
|
|
19301
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19302
|
+
this.recordError(resource.name, "list", error);
|
|
19303
|
+
throw error;
|
|
19304
|
+
}
|
|
19305
|
+
}.bind(this);
|
|
19306
|
+
}
|
|
19307
|
+
recordOperation(resourceName, operation, duration, isError) {
|
|
19308
|
+
if (this.metrics.operations[operation]) {
|
|
19309
|
+
this.metrics.operations[operation].count++;
|
|
19310
|
+
this.metrics.operations[operation].totalTime += duration;
|
|
19311
|
+
if (isError) {
|
|
19312
|
+
this.metrics.operations[operation].errors++;
|
|
19313
|
+
}
|
|
19314
|
+
}
|
|
19315
|
+
if (!this.metrics.resources[resourceName]) {
|
|
19316
|
+
this.metrics.resources[resourceName] = {
|
|
19317
|
+
insert: { count: 0, totalTime: 0, errors: 0 },
|
|
19318
|
+
update: { count: 0, totalTime: 0, errors: 0 },
|
|
19319
|
+
delete: { count: 0, totalTime: 0, errors: 0 },
|
|
19320
|
+
get: { count: 0, totalTime: 0, errors: 0 },
|
|
19321
|
+
list: { count: 0, totalTime: 0, errors: 0 },
|
|
19322
|
+
count: { count: 0, totalTime: 0, errors: 0 }
|
|
19323
|
+
};
|
|
19324
|
+
}
|
|
19325
|
+
if (this.metrics.resources[resourceName][operation]) {
|
|
19326
|
+
this.metrics.resources[resourceName][operation].count++;
|
|
19327
|
+
this.metrics.resources[resourceName][operation].totalTime += duration;
|
|
19328
|
+
if (isError) {
|
|
19329
|
+
this.metrics.resources[resourceName][operation].errors++;
|
|
19330
|
+
}
|
|
19331
|
+
}
|
|
19332
|
+
if (this.config.collectPerformance) {
|
|
19333
|
+
this.metrics.performance.push({
|
|
19334
|
+
resourceName,
|
|
19335
|
+
operation,
|
|
19336
|
+
duration,
|
|
19337
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
19338
|
+
});
|
|
19339
|
+
}
|
|
19340
|
+
}
|
|
19341
|
+
recordError(resourceName, operation, error) {
|
|
19342
|
+
if (!this.config.collectErrors) return;
|
|
19343
|
+
this.metrics.errors.push({
|
|
19344
|
+
resourceName,
|
|
19345
|
+
operation,
|
|
19346
|
+
error: error.message,
|
|
19347
|
+
stack: error.stack,
|
|
19348
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
19349
|
+
});
|
|
19350
|
+
}
|
|
19351
|
+
startFlushTimer() {
|
|
19352
|
+
if (this.flushTimer) {
|
|
19353
|
+
clearInterval(this.flushTimer);
|
|
19354
|
+
}
|
|
19355
|
+
if (this.config.flushInterval > 0) {
|
|
19356
|
+
this.flushTimer = setInterval(() => {
|
|
19357
|
+
this.flushMetrics().catch(console.error);
|
|
19358
|
+
}, this.config.flushInterval);
|
|
19359
|
+
}
|
|
19360
|
+
}
|
|
19361
|
+
async flushMetrics() {
|
|
19362
|
+
if (!this.metricsResource) return;
|
|
19363
|
+
try {
|
|
19364
|
+
const metadata = process.env.NODE_ENV === "test" ? {} : { global: "true" };
|
|
19365
|
+
const perfMetadata = process.env.NODE_ENV === "test" ? {} : { perf: "true" };
|
|
19366
|
+
const errorMetadata = process.env.NODE_ENV === "test" ? {} : { error: "true" };
|
|
19367
|
+
const resourceMetadata = process.env.NODE_ENV === "test" ? {} : { resource: "true" };
|
|
19368
|
+
for (const [operation, data] of Object.entries(this.metrics.operations)) {
|
|
19369
|
+
if (data.count > 0) {
|
|
19370
|
+
await this.metricsResource.insert({
|
|
19371
|
+
id: `metrics-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19372
|
+
type: "operation",
|
|
19373
|
+
resourceName: "global",
|
|
19374
|
+
operation,
|
|
19375
|
+
count: data.count,
|
|
19376
|
+
totalTime: data.totalTime,
|
|
19377
|
+
errors: data.errors,
|
|
19378
|
+
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
19379
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19380
|
+
metadata
|
|
19381
|
+
});
|
|
19382
|
+
}
|
|
19383
|
+
}
|
|
19384
|
+
for (const [resourceName, operations] of Object.entries(this.metrics.resources)) {
|
|
19385
|
+
for (const [operation, data] of Object.entries(operations)) {
|
|
19386
|
+
if (data.count > 0) {
|
|
19387
|
+
await this.metricsResource.insert({
|
|
19388
|
+
id: `metrics-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19389
|
+
type: "operation",
|
|
19390
|
+
resourceName,
|
|
19391
|
+
operation,
|
|
19392
|
+
count: data.count,
|
|
19393
|
+
totalTime: data.totalTime,
|
|
19394
|
+
errors: data.errors,
|
|
19395
|
+
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
19396
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19397
|
+
metadata: resourceMetadata
|
|
19398
|
+
});
|
|
19399
|
+
}
|
|
19400
|
+
}
|
|
19401
|
+
}
|
|
19402
|
+
if (this.config.collectPerformance && this.metrics.performance.length > 0) {
|
|
19403
|
+
for (const perf of this.metrics.performance) {
|
|
19404
|
+
await this.performanceResource.insert({
|
|
19405
|
+
id: `perf-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19406
|
+
resourceName: perf.resourceName,
|
|
19407
|
+
operation: perf.operation,
|
|
19408
|
+
duration: perf.duration,
|
|
19409
|
+
timestamp: perf.timestamp,
|
|
19410
|
+
metadata: perfMetadata
|
|
19411
|
+
});
|
|
19412
|
+
}
|
|
19413
|
+
}
|
|
19414
|
+
if (this.config.collectErrors && this.metrics.errors.length > 0) {
|
|
19415
|
+
for (const error of this.metrics.errors) {
|
|
19416
|
+
await this.errorsResource.insert({
|
|
19417
|
+
id: `error-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19418
|
+
resourceName: error.resourceName,
|
|
19419
|
+
operation: error.operation,
|
|
19420
|
+
error: error.error,
|
|
19421
|
+
stack: error.stack,
|
|
19422
|
+
timestamp: error.timestamp,
|
|
19423
|
+
metadata: errorMetadata
|
|
19424
|
+
});
|
|
19425
|
+
}
|
|
19426
|
+
}
|
|
19427
|
+
this.resetMetrics();
|
|
19428
|
+
} catch (error) {
|
|
19429
|
+
console.error("Failed to flush metrics:", error);
|
|
19430
|
+
}
|
|
19431
|
+
}
|
|
19432
|
+
resetMetrics() {
|
|
19433
|
+
for (const operation of Object.keys(this.metrics.operations)) {
|
|
19434
|
+
this.metrics.operations[operation] = { count: 0, totalTime: 0, errors: 0 };
|
|
19435
|
+
}
|
|
19436
|
+
for (const resourceName of Object.keys(this.metrics.resources)) {
|
|
19437
|
+
for (const operation of Object.keys(this.metrics.resources[resourceName])) {
|
|
19438
|
+
this.metrics.resources[resourceName][operation] = { count: 0, totalTime: 0, errors: 0 };
|
|
19439
|
+
}
|
|
19440
|
+
}
|
|
19441
|
+
this.metrics.performance = [];
|
|
19442
|
+
this.metrics.errors = [];
|
|
19443
|
+
}
|
|
19444
|
+
// Utility methods
|
|
19445
|
+
async getMetrics(options = {}) {
|
|
19446
|
+
const {
|
|
19447
|
+
type = "operation",
|
|
19448
|
+
resourceName,
|
|
19449
|
+
operation,
|
|
19450
|
+
startDate,
|
|
19451
|
+
endDate,
|
|
19452
|
+
limit = 100,
|
|
19453
|
+
offset = 0
|
|
19454
|
+
} = options;
|
|
19455
|
+
if (!this.metricsResource) return [];
|
|
19456
|
+
const allMetrics = await this.metricsResource.getAll();
|
|
19457
|
+
let filtered = allMetrics.filter((metric) => {
|
|
19458
|
+
if (type && metric.type !== type) return false;
|
|
19459
|
+
if (resourceName && metric.resourceName !== resourceName) return false;
|
|
19460
|
+
if (operation && metric.operation !== operation) return false;
|
|
19461
|
+
if (startDate && new Date(metric.timestamp) < new Date(startDate)) return false;
|
|
19462
|
+
if (endDate && new Date(metric.timestamp) > new Date(endDate)) return false;
|
|
19463
|
+
return true;
|
|
19464
|
+
});
|
|
19465
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
19466
|
+
return filtered.slice(offset, offset + limit);
|
|
19467
|
+
}
|
|
19468
|
+
async getErrorLogs(options = {}) {
|
|
19469
|
+
if (!this.errorsResource) return [];
|
|
19470
|
+
const {
|
|
19471
|
+
resourceName,
|
|
19472
|
+
operation,
|
|
19473
|
+
startDate,
|
|
19474
|
+
endDate,
|
|
19475
|
+
limit = 100,
|
|
19476
|
+
offset = 0
|
|
19477
|
+
} = options;
|
|
19478
|
+
const allErrors = await this.errorsResource.getAll();
|
|
19479
|
+
let filtered = allErrors.filter((error) => {
|
|
19480
|
+
if (resourceName && error.resourceName !== resourceName) return false;
|
|
19481
|
+
if (operation && error.operation !== operation) return false;
|
|
19482
|
+
if (startDate && new Date(error.timestamp) < new Date(startDate)) return false;
|
|
19483
|
+
if (endDate && new Date(error.timestamp) > new Date(endDate)) return false;
|
|
19484
|
+
return true;
|
|
19485
|
+
});
|
|
19486
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
19487
|
+
return filtered.slice(offset, offset + limit);
|
|
19488
|
+
}
|
|
19489
|
+
async getPerformanceLogs(options = {}) {
|
|
19490
|
+
if (!this.performanceResource) return [];
|
|
19491
|
+
const {
|
|
19492
|
+
resourceName,
|
|
19493
|
+
operation,
|
|
19494
|
+
startDate,
|
|
19495
|
+
endDate,
|
|
19496
|
+
limit = 100,
|
|
19497
|
+
offset = 0
|
|
19498
|
+
} = options;
|
|
19499
|
+
const allPerformance = await this.performanceResource.getAll();
|
|
19500
|
+
let filtered = allPerformance.filter((perf) => {
|
|
19501
|
+
if (resourceName && perf.resourceName !== resourceName) return false;
|
|
19502
|
+
if (operation && perf.operation !== operation) return false;
|
|
19503
|
+
if (startDate && new Date(perf.timestamp) < new Date(startDate)) return false;
|
|
19504
|
+
if (endDate && new Date(perf.timestamp) > new Date(endDate)) return false;
|
|
19505
|
+
return true;
|
|
19506
|
+
});
|
|
19507
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
19508
|
+
return filtered.slice(offset, offset + limit);
|
|
19509
|
+
}
|
|
19510
|
+
async getStats() {
|
|
19511
|
+
const now = /* @__PURE__ */ new Date();
|
|
19512
|
+
const startDate = new Date(now.getTime() - 24 * 60 * 60 * 1e3);
|
|
19513
|
+
const [metrics, errors, performance] = await Promise.all([
|
|
19514
|
+
this.getMetrics({ startDate: startDate.toISOString() }),
|
|
19515
|
+
this.getErrorLogs({ startDate: startDate.toISOString() }),
|
|
19516
|
+
this.getPerformanceLogs({ startDate: startDate.toISOString() })
|
|
19517
|
+
]);
|
|
19518
|
+
const stats = {
|
|
19519
|
+
period: "24h",
|
|
19520
|
+
totalOperations: 0,
|
|
19521
|
+
totalErrors: errors.length,
|
|
19522
|
+
avgResponseTime: 0,
|
|
19523
|
+
operationsByType: {},
|
|
19524
|
+
resources: {},
|
|
19525
|
+
uptime: {
|
|
19526
|
+
startTime: this.metrics.startTime,
|
|
19527
|
+
duration: now.getTime() - new Date(this.metrics.startTime).getTime()
|
|
19528
|
+
}
|
|
19529
|
+
};
|
|
19530
|
+
for (const metric of metrics) {
|
|
19531
|
+
if (metric.type === "operation") {
|
|
19532
|
+
stats.totalOperations += metric.count;
|
|
19533
|
+
if (!stats.operationsByType[metric.operation]) {
|
|
19534
|
+
stats.operationsByType[metric.operation] = {
|
|
19535
|
+
count: 0,
|
|
19536
|
+
errors: 0,
|
|
19537
|
+
avgTime: 0
|
|
19538
|
+
};
|
|
19539
|
+
}
|
|
19540
|
+
stats.operationsByType[metric.operation].count += metric.count;
|
|
19541
|
+
stats.operationsByType[metric.operation].errors += metric.errors;
|
|
19542
|
+
const current = stats.operationsByType[metric.operation];
|
|
19543
|
+
const totalCount2 = current.count;
|
|
19544
|
+
const newAvg = (current.avgTime * (totalCount2 - metric.count) + metric.totalTime) / totalCount2;
|
|
19545
|
+
current.avgTime = newAvg;
|
|
19546
|
+
}
|
|
19547
|
+
}
|
|
19548
|
+
const totalTime = metrics.reduce((sum, m) => sum + m.totalTime, 0);
|
|
19549
|
+
const totalCount = metrics.reduce((sum, m) => sum + m.count, 0);
|
|
19550
|
+
stats.avgResponseTime = totalCount > 0 ? totalTime / totalCount : 0;
|
|
19551
|
+
return stats;
|
|
19552
|
+
}
|
|
19553
|
+
async cleanupOldData() {
|
|
19554
|
+
const cutoffDate = /* @__PURE__ */ new Date();
|
|
19555
|
+
cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
|
|
19556
|
+
if (this.metricsResource) {
|
|
19557
|
+
const oldMetrics = await this.getMetrics({ endDate: cutoffDate.toISOString() });
|
|
19558
|
+
for (const metric of oldMetrics) {
|
|
19559
|
+
await this.metricsResource.delete(metric.id);
|
|
19560
|
+
}
|
|
19561
|
+
}
|
|
19562
|
+
if (this.errorsResource) {
|
|
19563
|
+
const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
|
|
19564
|
+
for (const error of oldErrors) {
|
|
19565
|
+
await this.errorsResource.delete(error.id);
|
|
19566
|
+
}
|
|
19567
|
+
}
|
|
19568
|
+
if (this.performanceResource) {
|
|
19569
|
+
const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
|
|
19570
|
+
for (const perf of oldPerformance) {
|
|
19571
|
+
await this.performanceResource.delete(perf.id);
|
|
19572
|
+
}
|
|
19573
|
+
}
|
|
19574
|
+
console.log(`Cleaned up data older than ${this.config.retentionDays} days`);
|
|
19575
|
+
}
|
|
19576
|
+
}
|
|
19577
|
+
|
|
19578
|
+
class BaseReplicator extends EventEmitter {
|
|
19579
|
+
constructor(config = {}) {
|
|
19580
|
+
super();
|
|
19581
|
+
this.config = config;
|
|
19582
|
+
this.name = this.constructor.name;
|
|
19583
|
+
this.enabled = config.enabled !== false;
|
|
19584
|
+
}
|
|
19585
|
+
/**
|
|
19586
|
+
* Initialize the replicator
|
|
19587
|
+
* @param {Object} database - The s3db database instance
|
|
19588
|
+
* @returns {Promise<void>}
|
|
19589
|
+
*/
|
|
19590
|
+
async initialize(database) {
|
|
19591
|
+
this.database = database;
|
|
19592
|
+
this.emit("initialized", { replicator: this.name });
|
|
19593
|
+
}
|
|
19594
|
+
/**
|
|
19595
|
+
* Replicate data to the target
|
|
19596
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
19597
|
+
* @param {string} operation - Operation type (insert, update, delete)
|
|
19598
|
+
* @param {Object} data - The data to replicate
|
|
19599
|
+
* @param {string} id - Record ID
|
|
19600
|
+
* @returns {Promise<Object>} Replication result
|
|
19601
|
+
*/
|
|
19602
|
+
async replicate(resourceName, operation, data, id) {
|
|
19603
|
+
throw new Error(`replicate() method must be implemented by ${this.name}`);
|
|
19604
|
+
}
|
|
19605
|
+
/**
|
|
19606
|
+
* Replicate multiple records in batch
|
|
19607
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
19608
|
+
* @param {Array} records - Array of records to replicate
|
|
19609
|
+
* @returns {Promise<Object>} Batch replication result
|
|
19610
|
+
*/
|
|
19611
|
+
async replicateBatch(resourceName, records) {
|
|
19612
|
+
throw new Error(`replicateBatch() method must be implemented by ${this.name}`);
|
|
19613
|
+
}
|
|
19614
|
+
/**
|
|
19615
|
+
* Test the connection to the target
|
|
19616
|
+
* @returns {Promise<boolean>} True if connection is successful
|
|
19617
|
+
*/
|
|
19618
|
+
async testConnection() {
|
|
19619
|
+
throw new Error(`testConnection() method must be implemented by ${this.name}`);
|
|
19620
|
+
}
|
|
19621
|
+
/**
|
|
19622
|
+
* Get replicator status and statistics
|
|
19623
|
+
* @returns {Promise<Object>} Status information
|
|
19624
|
+
*/
|
|
19625
|
+
async getStatus() {
|
|
19626
|
+
return {
|
|
19627
|
+
name: this.name,
|
|
19628
|
+
enabled: this.enabled,
|
|
19629
|
+
config: this.config,
|
|
19630
|
+
connected: false
|
|
19631
|
+
};
|
|
19632
|
+
}
|
|
19633
|
+
/**
|
|
19634
|
+
* Cleanup resources
|
|
19635
|
+
* @returns {Promise<void>}
|
|
19636
|
+
*/
|
|
19637
|
+
async cleanup() {
|
|
19638
|
+
this.emit("cleanup", { replicator: this.name });
|
|
19639
|
+
}
|
|
19640
|
+
/**
|
|
19641
|
+
* Validate replicator configuration
|
|
19642
|
+
* @returns {Object} Validation result
|
|
19643
|
+
*/
|
|
19644
|
+
validateConfig() {
|
|
19645
|
+
return { isValid: true, errors: [] };
|
|
19646
|
+
}
|
|
19647
|
+
}
|
|
19648
|
+
|
|
19649
|
+
class S3dbReplicator extends BaseReplicator {
|
|
19650
|
+
constructor(config = {}, resources = []) {
|
|
19651
|
+
super(config);
|
|
19652
|
+
this.resources = resources;
|
|
19653
|
+
this.connectionString = config.connectionString;
|
|
19654
|
+
this.region = config.region;
|
|
19655
|
+
this.bucket = config.bucket;
|
|
19656
|
+
this.keyPrefix = config.keyPrefix;
|
|
19657
|
+
}
|
|
19658
|
+
validateConfig() {
|
|
19659
|
+
const errors = [];
|
|
19660
|
+
if (!this.connectionString && !this.bucket) {
|
|
19661
|
+
errors.push("Either connectionString or bucket must be provided");
|
|
19662
|
+
}
|
|
19663
|
+
return {
|
|
19664
|
+
isValid: errors.length === 0,
|
|
19665
|
+
errors
|
|
19666
|
+
};
|
|
19667
|
+
}
|
|
19668
|
+
async initialize(database) {
|
|
19669
|
+
await super.initialize(database);
|
|
19670
|
+
const targetConfig = {
|
|
19671
|
+
connectionString: this.connectionString,
|
|
19672
|
+
region: this.region,
|
|
19673
|
+
bucket: this.bucket,
|
|
19674
|
+
keyPrefix: this.keyPrefix,
|
|
19675
|
+
verbose: this.config.verbose || false
|
|
19676
|
+
};
|
|
19677
|
+
this.targetDatabase = new S3db(targetConfig);
|
|
19678
|
+
await this.targetDatabase.connect();
|
|
19679
|
+
this.emit("connected", {
|
|
19680
|
+
replicator: this.name,
|
|
19681
|
+
target: this.connectionString || this.bucket
|
|
19682
|
+
});
|
|
19683
|
+
}
|
|
19684
|
+
async replicate(resourceName, operation, data, id) {
|
|
19685
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19686
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19687
|
+
}
|
|
19688
|
+
try {
|
|
19689
|
+
let result;
|
|
19690
|
+
switch (operation) {
|
|
19691
|
+
case "insert":
|
|
19692
|
+
result = await this.targetDatabase.resources[resourceName]?.insert(data);
|
|
19693
|
+
break;
|
|
19694
|
+
case "update":
|
|
19695
|
+
result = await this.targetDatabase.resources[resourceName]?.update(id, data);
|
|
19696
|
+
break;
|
|
19697
|
+
case "delete":
|
|
19698
|
+
result = await this.targetDatabase.resources[resourceName]?.delete(id);
|
|
19699
|
+
break;
|
|
19700
|
+
default:
|
|
19701
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
19702
|
+
}
|
|
19703
|
+
this.emit("replicated", {
|
|
19704
|
+
replicator: this.name,
|
|
19705
|
+
resourceName,
|
|
19706
|
+
operation,
|
|
19707
|
+
id,
|
|
19708
|
+
success: true
|
|
19709
|
+
});
|
|
19710
|
+
return { success: true, result };
|
|
19711
|
+
} catch (error) {
|
|
19712
|
+
this.emit("replication_error", {
|
|
19713
|
+
replicator: this.name,
|
|
19714
|
+
resourceName,
|
|
19715
|
+
operation,
|
|
19716
|
+
id,
|
|
19717
|
+
error: error.message
|
|
19718
|
+
});
|
|
19719
|
+
return { success: false, error: error.message };
|
|
19720
|
+
}
|
|
19721
|
+
}
|
|
19722
|
+
async replicateBatch(resourceName, records) {
|
|
19723
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19724
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19725
|
+
}
|
|
19726
|
+
try {
|
|
19727
|
+
const results = [];
|
|
19728
|
+
const errors = [];
|
|
19729
|
+
for (const record of records) {
|
|
19730
|
+
try {
|
|
19731
|
+
const result = await this.replicate(
|
|
19732
|
+
resourceName,
|
|
19733
|
+
record.operation,
|
|
19734
|
+
record.data,
|
|
19735
|
+
record.id
|
|
19736
|
+
);
|
|
19737
|
+
results.push(result);
|
|
19738
|
+
} catch (error) {
|
|
19739
|
+
errors.push({ id: record.id, error: error.message });
|
|
19740
|
+
}
|
|
19741
|
+
}
|
|
19742
|
+
this.emit("batch_replicated", {
|
|
19743
|
+
replicator: this.name,
|
|
19744
|
+
resourceName,
|
|
19745
|
+
total: records.length,
|
|
19746
|
+
successful: results.filter((r) => r.success).length,
|
|
19747
|
+
errors: errors.length
|
|
19748
|
+
});
|
|
19749
|
+
return {
|
|
19750
|
+
success: errors.length === 0,
|
|
19751
|
+
results,
|
|
19752
|
+
errors,
|
|
19753
|
+
total: records.length
|
|
19754
|
+
};
|
|
19755
|
+
} catch (error) {
|
|
19756
|
+
this.emit("batch_replication_error", {
|
|
19757
|
+
replicator: this.name,
|
|
19758
|
+
resourceName,
|
|
19759
|
+
error: error.message
|
|
19760
|
+
});
|
|
19761
|
+
return { success: false, error: error.message };
|
|
19762
|
+
}
|
|
19763
|
+
}
|
|
19764
|
+
async testConnection() {
|
|
19765
|
+
try {
|
|
19766
|
+
if (!this.targetDatabase) {
|
|
19767
|
+
await this.initialize(this.database);
|
|
19768
|
+
}
|
|
19769
|
+
await this.targetDatabase.listResources();
|
|
19770
|
+
return true;
|
|
19771
|
+
} catch (error) {
|
|
19772
|
+
this.emit("connection_error", {
|
|
19773
|
+
replicator: this.name,
|
|
19774
|
+
error: error.message
|
|
19775
|
+
});
|
|
19776
|
+
return false;
|
|
19777
|
+
}
|
|
19778
|
+
}
|
|
19779
|
+
async getStatus() {
|
|
19780
|
+
const baseStatus = await super.getStatus();
|
|
19781
|
+
return {
|
|
19782
|
+
...baseStatus,
|
|
19783
|
+
connected: !!this.targetDatabase,
|
|
19784
|
+
targetDatabase: this.connectionString || this.bucket,
|
|
19785
|
+
resources: this.resources,
|
|
19786
|
+
totalReplications: this.listenerCount("replicated"),
|
|
19787
|
+
totalErrors: this.listenerCount("replication_error")
|
|
19788
|
+
};
|
|
19789
|
+
}
|
|
19790
|
+
async cleanup() {
|
|
19791
|
+
if (this.targetDatabase) {
|
|
19792
|
+
this.targetDatabase.removeAllListeners();
|
|
19793
|
+
}
|
|
19794
|
+
await super.cleanup();
|
|
19795
|
+
}
|
|
19796
|
+
shouldReplicateResource(resourceName) {
|
|
19797
|
+
return this.resources.length === 0 || this.resources.includes(resourceName);
|
|
19798
|
+
}
|
|
19799
|
+
}
|
|
19800
|
+
|
|
19801
|
+
class SqsReplicator extends BaseReplicator {
|
|
19802
|
+
constructor(config = {}, resources = []) {
|
|
19803
|
+
super(config);
|
|
19804
|
+
this.resources = resources;
|
|
19805
|
+
this.queueUrl = config.queueUrl;
|
|
19806
|
+
this.queues = config.queues || {};
|
|
19807
|
+
this.defaultQueueUrl = config.defaultQueueUrl;
|
|
19808
|
+
this.region = config.region || "us-east-1";
|
|
19809
|
+
this.sqsClient = null;
|
|
19810
|
+
this.messageGroupId = config.messageGroupId;
|
|
19811
|
+
this.deduplicationId = config.deduplicationId;
|
|
19812
|
+
}
|
|
19813
|
+
validateConfig() {
|
|
19814
|
+
const errors = [];
|
|
19815
|
+
if (!this.queueUrl && Object.keys(this.queues).length === 0 && !this.defaultQueueUrl) {
|
|
19816
|
+
errors.push("Either queueUrl, queues object, or defaultQueueUrl must be provided");
|
|
19817
|
+
}
|
|
19818
|
+
return {
|
|
19819
|
+
isValid: errors.length === 0,
|
|
19820
|
+
errors
|
|
19821
|
+
};
|
|
19822
|
+
}
|
|
19823
|
+
/**
|
|
19824
|
+
* Get the appropriate queue URL for a resource
|
|
19825
|
+
*/
|
|
19826
|
+
getQueueUrlForResource(resourceName) {
|
|
19827
|
+
if (this.queues[resourceName]) {
|
|
19828
|
+
return this.queues[resourceName];
|
|
19829
|
+
}
|
|
19830
|
+
if (this.queueUrl) {
|
|
19831
|
+
return this.queueUrl;
|
|
19832
|
+
}
|
|
19833
|
+
if (this.defaultQueueUrl) {
|
|
19834
|
+
return this.defaultQueueUrl;
|
|
19835
|
+
}
|
|
19836
|
+
throw new Error(`No queue URL found for resource '${resourceName}'`);
|
|
19837
|
+
}
|
|
19838
|
+
/**
|
|
19839
|
+
* Create standardized message structure
|
|
19840
|
+
*/
|
|
19841
|
+
createMessage(resourceName, operation, data, id, beforeData = null) {
|
|
19842
|
+
const baseMessage = {
|
|
19843
|
+
resource: resourceName,
|
|
19844
|
+
action: operation,
|
|
19845
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19846
|
+
source: "s3db-replication"
|
|
19847
|
+
};
|
|
19848
|
+
switch (operation) {
|
|
19849
|
+
case "insert":
|
|
19850
|
+
return {
|
|
19851
|
+
...baseMessage,
|
|
19852
|
+
data
|
|
19853
|
+
};
|
|
19854
|
+
case "update":
|
|
19855
|
+
return {
|
|
19856
|
+
...baseMessage,
|
|
19857
|
+
before: beforeData,
|
|
19858
|
+
data
|
|
19859
|
+
};
|
|
19860
|
+
case "delete":
|
|
19861
|
+
return {
|
|
19862
|
+
...baseMessage,
|
|
19863
|
+
data
|
|
19864
|
+
};
|
|
19865
|
+
default:
|
|
19866
|
+
return {
|
|
19867
|
+
...baseMessage,
|
|
19868
|
+
data
|
|
19869
|
+
};
|
|
19870
|
+
}
|
|
19871
|
+
}
|
|
19872
|
+
async initialize(database) {
|
|
19873
|
+
await super.initialize(database);
|
|
19874
|
+
try {
|
|
19875
|
+
const { SQSClient, SendMessageCommand, SendMessageBatchCommand } = await import('@aws-sdk/client-sqs');
|
|
19876
|
+
this.sqsClient = new SQSClient({
|
|
19877
|
+
region: this.region,
|
|
19878
|
+
credentials: this.config.credentials
|
|
19879
|
+
});
|
|
19880
|
+
this.emit("initialized", {
|
|
19881
|
+
replicator: this.name,
|
|
19882
|
+
queueUrl: this.queueUrl,
|
|
19883
|
+
queues: this.queues,
|
|
19884
|
+
defaultQueueUrl: this.defaultQueueUrl
|
|
19885
|
+
});
|
|
19886
|
+
} catch (error) {
|
|
19887
|
+
this.emit("initialization_error", {
|
|
19888
|
+
replicator: this.name,
|
|
19889
|
+
error: error.message
|
|
19890
|
+
});
|
|
19891
|
+
throw error;
|
|
19892
|
+
}
|
|
19893
|
+
}
|
|
19894
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
19895
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19896
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19897
|
+
}
|
|
19898
|
+
try {
|
|
19899
|
+
const { SendMessageCommand } = await import('@aws-sdk/client-sqs');
|
|
19900
|
+
const queueUrl = this.getQueueUrlForResource(resourceName);
|
|
19901
|
+
const message = this.createMessage(resourceName, operation, data, id, beforeData);
|
|
19902
|
+
const command = new SendMessageCommand({
|
|
19903
|
+
QueueUrl: queueUrl,
|
|
19904
|
+
MessageBody: JSON.stringify(message),
|
|
19905
|
+
MessageGroupId: this.messageGroupId,
|
|
19906
|
+
MessageDeduplicationId: this.deduplicationId ? `${resourceName}:${operation}:${id}` : void 0
|
|
19907
|
+
});
|
|
19908
|
+
const result = await this.sqsClient.send(command);
|
|
19909
|
+
this.emit("replicated", {
|
|
19910
|
+
replicator: this.name,
|
|
19911
|
+
resourceName,
|
|
19912
|
+
operation,
|
|
19913
|
+
id,
|
|
19914
|
+
queueUrl,
|
|
19915
|
+
messageId: result.MessageId,
|
|
19916
|
+
success: true
|
|
19917
|
+
});
|
|
19918
|
+
return { success: true, messageId: result.MessageId, queueUrl };
|
|
19919
|
+
} catch (error) {
|
|
19920
|
+
this.emit("replication_error", {
|
|
19921
|
+
replicator: this.name,
|
|
19922
|
+
resourceName,
|
|
19923
|
+
operation,
|
|
19924
|
+
id,
|
|
19925
|
+
error: error.message
|
|
19926
|
+
});
|
|
19927
|
+
return { success: false, error: error.message };
|
|
19928
|
+
}
|
|
19929
|
+
}
|
|
19930
|
+
async replicateBatch(resourceName, records) {
|
|
19931
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19932
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19933
|
+
}
|
|
19934
|
+
try {
|
|
19935
|
+
const { SendMessageBatchCommand } = await import('@aws-sdk/client-sqs');
|
|
19936
|
+
const queueUrl = this.getQueueUrlForResource(resourceName);
|
|
19937
|
+
const batchSize = 10;
|
|
19938
|
+
const batches = [];
|
|
19939
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
19940
|
+
batches.push(records.slice(i, i + batchSize));
|
|
19941
|
+
}
|
|
19942
|
+
const results = [];
|
|
19943
|
+
const errors = [];
|
|
19944
|
+
for (const batch of batches) {
|
|
19945
|
+
try {
|
|
19946
|
+
const entries = batch.map((record, index) => ({
|
|
19947
|
+
Id: `${record.id}-${index}`,
|
|
19948
|
+
MessageBody: JSON.stringify(this.createMessage(
|
|
19949
|
+
resourceName,
|
|
19950
|
+
record.operation,
|
|
19951
|
+
record.data,
|
|
19952
|
+
record.id,
|
|
19953
|
+
record.beforeData
|
|
19954
|
+
)),
|
|
19955
|
+
MessageGroupId: this.messageGroupId,
|
|
19956
|
+
MessageDeduplicationId: this.deduplicationId ? `${resourceName}:${record.operation}:${record.id}` : void 0
|
|
19957
|
+
}));
|
|
19958
|
+
const command = new SendMessageBatchCommand({
|
|
19959
|
+
QueueUrl: queueUrl,
|
|
19960
|
+
Entries: entries
|
|
19961
|
+
});
|
|
19962
|
+
const result = await this.sqsClient.send(command);
|
|
19963
|
+
results.push(result);
|
|
19964
|
+
} catch (error) {
|
|
19965
|
+
errors.push({ batch: batch.length, error: error.message });
|
|
19966
|
+
}
|
|
19967
|
+
}
|
|
19968
|
+
this.emit("batch_replicated", {
|
|
19969
|
+
replicator: this.name,
|
|
19970
|
+
resourceName,
|
|
19971
|
+
queueUrl,
|
|
19972
|
+
total: records.length,
|
|
19973
|
+
successful: results.length,
|
|
19974
|
+
errors: errors.length
|
|
19975
|
+
});
|
|
19976
|
+
return {
|
|
19977
|
+
success: errors.length === 0,
|
|
19978
|
+
results,
|
|
19979
|
+
errors,
|
|
19980
|
+
total: records.length,
|
|
19981
|
+
queueUrl
|
|
19982
|
+
};
|
|
19983
|
+
} catch (error) {
|
|
19984
|
+
this.emit("batch_replication_error", {
|
|
19985
|
+
replicator: this.name,
|
|
19986
|
+
resourceName,
|
|
19987
|
+
error: error.message
|
|
19988
|
+
});
|
|
19989
|
+
return { success: false, error: error.message };
|
|
19990
|
+
}
|
|
19991
|
+
}
|
|
19992
|
+
async testConnection() {
|
|
19993
|
+
try {
|
|
19994
|
+
if (!this.sqsClient) {
|
|
19995
|
+
await this.initialize(this.database);
|
|
19996
|
+
}
|
|
19997
|
+
const { GetQueueAttributesCommand } = await import('@aws-sdk/client-sqs');
|
|
19998
|
+
const command = new GetQueueAttributesCommand({
|
|
19999
|
+
QueueUrl: this.queueUrl,
|
|
20000
|
+
AttributeNames: ["QueueArn"]
|
|
20001
|
+
});
|
|
20002
|
+
await this.sqsClient.send(command);
|
|
20003
|
+
return true;
|
|
20004
|
+
} catch (error) {
|
|
20005
|
+
this.emit("connection_error", {
|
|
20006
|
+
replicator: this.name,
|
|
20007
|
+
error: error.message
|
|
20008
|
+
});
|
|
20009
|
+
return false;
|
|
20010
|
+
}
|
|
20011
|
+
}
|
|
20012
|
+
async getStatus() {
|
|
20013
|
+
const baseStatus = await super.getStatus();
|
|
20014
|
+
return {
|
|
20015
|
+
...baseStatus,
|
|
20016
|
+
connected: !!this.sqsClient,
|
|
20017
|
+
queueUrl: this.queueUrl,
|
|
20018
|
+
region: this.region,
|
|
20019
|
+
resources: this.resources,
|
|
20020
|
+
totalReplications: this.listenerCount("replicated"),
|
|
20021
|
+
totalErrors: this.listenerCount("replication_error")
|
|
20022
|
+
};
|
|
20023
|
+
}
|
|
20024
|
+
async cleanup() {
|
|
20025
|
+
if (this.sqsClient) {
|
|
20026
|
+
this.sqsClient.destroy();
|
|
20027
|
+
}
|
|
20028
|
+
await super.cleanup();
|
|
20029
|
+
}
|
|
20030
|
+
shouldReplicateResource(resourceName) {
|
|
20031
|
+
return this.resources.length === 0 || this.resources.includes(resourceName);
|
|
20032
|
+
}
|
|
20033
|
+
}
|
|
20034
|
+
|
|
20035
|
+
class BigqueryReplicator extends BaseReplicator {
|
|
20036
|
+
constructor(config = {}, resources = {}) {
|
|
20037
|
+
super(config);
|
|
20038
|
+
this.projectId = config.projectId;
|
|
20039
|
+
this.datasetId = config.datasetId;
|
|
20040
|
+
this.bigqueryClient = null;
|
|
20041
|
+
this.credentials = config.credentials;
|
|
20042
|
+
this.location = config.location || "US";
|
|
20043
|
+
this.logTable = config.logTable;
|
|
20044
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
20045
|
+
}
|
|
20046
|
+
parseResourcesConfig(resources) {
|
|
20047
|
+
const parsed = {};
|
|
20048
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
20049
|
+
if (typeof config === "string") {
|
|
20050
|
+
parsed[resourceName] = [{
|
|
20051
|
+
table: config,
|
|
20052
|
+
actions: ["insert"]
|
|
20053
|
+
}];
|
|
20054
|
+
} else if (Array.isArray(config)) {
|
|
20055
|
+
parsed[resourceName] = config.map((item) => {
|
|
20056
|
+
if (typeof item === "string") {
|
|
20057
|
+
return { table: item, actions: ["insert"] };
|
|
20058
|
+
}
|
|
20059
|
+
return {
|
|
20060
|
+
table: item.table,
|
|
20061
|
+
actions: item.actions || ["insert"]
|
|
20062
|
+
};
|
|
20063
|
+
});
|
|
20064
|
+
} else if (typeof config === "object") {
|
|
20065
|
+
parsed[resourceName] = [{
|
|
20066
|
+
table: config.table,
|
|
20067
|
+
actions: config.actions || ["insert"]
|
|
20068
|
+
}];
|
|
20069
|
+
}
|
|
20070
|
+
}
|
|
20071
|
+
return parsed;
|
|
20072
|
+
}
|
|
20073
|
+
validateConfig() {
|
|
20074
|
+
const errors = [];
|
|
20075
|
+
if (!this.projectId) errors.push("projectId is required");
|
|
20076
|
+
if (!this.datasetId) errors.push("datasetId is required");
|
|
20077
|
+
if (Object.keys(this.resources).length === 0) errors.push("At least one resource must be configured");
|
|
20078
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
20079
|
+
for (const tableConfig of tables) {
|
|
20080
|
+
if (!tableConfig.table) {
|
|
20081
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
20082
|
+
}
|
|
20083
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
20084
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
20085
|
+
}
|
|
20086
|
+
const validActions = ["insert", "update", "delete"];
|
|
20087
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
20088
|
+
if (invalidActions.length > 0) {
|
|
20089
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
20090
|
+
}
|
|
20091
|
+
}
|
|
20092
|
+
}
|
|
20093
|
+
return { isValid: errors.length === 0, errors };
|
|
20094
|
+
}
|
|
20095
|
+
async initialize(database) {
|
|
20096
|
+
await super.initialize(database);
|
|
20097
|
+
try {
|
|
20098
|
+
const { BigQuery } = await import('@google-cloud/bigquery');
|
|
20099
|
+
this.bigqueryClient = new BigQuery({
|
|
20100
|
+
projectId: this.projectId,
|
|
20101
|
+
credentials: this.credentials,
|
|
20102
|
+
location: this.location
|
|
20103
|
+
});
|
|
20104
|
+
this.emit("initialized", {
|
|
20105
|
+
replicator: this.name,
|
|
20106
|
+
projectId: this.projectId,
|
|
20107
|
+
datasetId: this.datasetId,
|
|
20108
|
+
resources: Object.keys(this.resources)
|
|
20109
|
+
});
|
|
20110
|
+
} catch (error) {
|
|
20111
|
+
this.emit("initialization_error", { replicator: this.name, error: error.message });
|
|
20112
|
+
throw error;
|
|
20113
|
+
}
|
|
20114
|
+
}
|
|
20115
|
+
shouldReplicateResource(resourceName) {
|
|
20116
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
20117
|
+
}
|
|
20118
|
+
shouldReplicateAction(resourceName, operation) {
|
|
20119
|
+
if (!this.resources[resourceName]) return false;
|
|
20120
|
+
return this.resources[resourceName].some(
|
|
20121
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
20122
|
+
);
|
|
20123
|
+
}
|
|
20124
|
+
getTablesForResource(resourceName, operation) {
|
|
20125
|
+
if (!this.resources[resourceName]) return [];
|
|
20126
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => tableConfig.table);
|
|
20127
|
+
}
|
|
20128
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
20129
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
20130
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
20131
|
+
}
|
|
20132
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
20133
|
+
return { skipped: true, reason: "action_not_included" };
|
|
20134
|
+
}
|
|
20135
|
+
const tables = this.getTablesForResource(resourceName, operation);
|
|
20136
|
+
if (tables.length === 0) {
|
|
20137
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
20138
|
+
}
|
|
20139
|
+
const results = [];
|
|
20140
|
+
const errors = [];
|
|
20141
|
+
try {
|
|
20142
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
20143
|
+
for (const tableId of tables) {
|
|
20144
|
+
try {
|
|
20145
|
+
const table = dataset.table(tableId);
|
|
20146
|
+
let job;
|
|
20147
|
+
if (operation === "insert") {
|
|
20148
|
+
const row = { ...data };
|
|
20149
|
+
job = await table.insert([row]);
|
|
20150
|
+
} else if (operation === "update") {
|
|
20151
|
+
const keys = Object.keys(data).filter((k) => k !== "id");
|
|
20152
|
+
const setClause = keys.map((k) => `${k}=@${k}`).join(", ");
|
|
20153
|
+
const params = { id };
|
|
20154
|
+
keys.forEach((k) => {
|
|
20155
|
+
params[k] = data[k];
|
|
20156
|
+
});
|
|
20157
|
+
const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableId}\` SET ${setClause} WHERE id=@id`;
|
|
20158
|
+
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
20159
|
+
query,
|
|
20160
|
+
params
|
|
20161
|
+
});
|
|
20162
|
+
await updateJob.getQueryResults();
|
|
20163
|
+
job = [updateJob];
|
|
20164
|
+
} else if (operation === "delete") {
|
|
20165
|
+
const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableId}\` WHERE id=@id`;
|
|
20166
|
+
const [deleteJob] = await this.bigqueryClient.createQueryJob({
|
|
20167
|
+
query,
|
|
20168
|
+
params: { id }
|
|
20169
|
+
});
|
|
20170
|
+
await deleteJob.getQueryResults();
|
|
20171
|
+
job = [deleteJob];
|
|
20172
|
+
} else {
|
|
20173
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
20174
|
+
}
|
|
20175
|
+
results.push({
|
|
20176
|
+
table: tableId,
|
|
20177
|
+
success: true,
|
|
20178
|
+
jobId: job[0]?.id
|
|
20179
|
+
});
|
|
20180
|
+
} catch (error) {
|
|
20181
|
+
errors.push({
|
|
20182
|
+
table: tableId,
|
|
20183
|
+
error: error.message
|
|
20184
|
+
});
|
|
20185
|
+
}
|
|
20186
|
+
}
|
|
20187
|
+
if (this.logTable) {
|
|
20188
|
+
try {
|
|
20189
|
+
const logTable = dataset.table(this.logTable);
|
|
20190
|
+
await logTable.insert([{
|
|
20191
|
+
resource_name: resourceName,
|
|
20192
|
+
operation,
|
|
20193
|
+
record_id: id,
|
|
20194
|
+
data: JSON.stringify(data),
|
|
20195
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20196
|
+
source: "s3db-replication"
|
|
20197
|
+
}]);
|
|
20198
|
+
} catch (error) {
|
|
20199
|
+
console.warn(`Failed to log operation to ${this.logTable}:`, error.message);
|
|
20200
|
+
}
|
|
20201
|
+
}
|
|
20202
|
+
const success = errors.length === 0;
|
|
20203
|
+
this.emit("replicated", {
|
|
20204
|
+
replicator: this.name,
|
|
20205
|
+
resourceName,
|
|
20206
|
+
operation,
|
|
20207
|
+
id,
|
|
20208
|
+
tables,
|
|
20209
|
+
results,
|
|
20210
|
+
errors,
|
|
20211
|
+
success
|
|
20212
|
+
});
|
|
20213
|
+
return {
|
|
20214
|
+
success,
|
|
20215
|
+
results,
|
|
20216
|
+
errors,
|
|
20217
|
+
tables
|
|
20218
|
+
};
|
|
20219
|
+
} catch (error) {
|
|
20220
|
+
this.emit("replication_error", {
|
|
20221
|
+
replicator: this.name,
|
|
20222
|
+
resourceName,
|
|
20223
|
+
operation,
|
|
20224
|
+
id,
|
|
20225
|
+
error: error.message
|
|
20226
|
+
});
|
|
20227
|
+
return { success: false, error: error.message };
|
|
20228
|
+
}
|
|
20229
|
+
}
|
|
20230
|
+
async replicateBatch(resourceName, records) {
|
|
20231
|
+
const results = [];
|
|
20232
|
+
const errors = [];
|
|
20233
|
+
for (const record of records) {
|
|
20234
|
+
try {
|
|
20235
|
+
const res = await this.replicate(
|
|
20236
|
+
resourceName,
|
|
20237
|
+
record.operation,
|
|
20238
|
+
record.data,
|
|
20239
|
+
record.id,
|
|
20240
|
+
record.beforeData
|
|
20241
|
+
);
|
|
20242
|
+
results.push(res);
|
|
20243
|
+
} catch (err) {
|
|
20244
|
+
errors.push({ id: record.id, error: err.message });
|
|
20245
|
+
}
|
|
20246
|
+
}
|
|
20247
|
+
return {
|
|
20248
|
+
success: errors.length === 0,
|
|
20249
|
+
results,
|
|
20250
|
+
errors
|
|
20251
|
+
};
|
|
20252
|
+
}
|
|
20253
|
+
async testConnection() {
|
|
20254
|
+
try {
|
|
20255
|
+
if (!this.bigqueryClient) await this.initialize();
|
|
20256
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
20257
|
+
await dataset.getMetadata();
|
|
20258
|
+
return true;
|
|
20259
|
+
} catch (error) {
|
|
20260
|
+
this.emit("connection_error", { replicator: this.name, error: error.message });
|
|
20261
|
+
return false;
|
|
20262
|
+
}
|
|
20263
|
+
}
|
|
20264
|
+
async cleanup() {
|
|
20265
|
+
}
|
|
20266
|
+
getStatus() {
|
|
20267
|
+
return {
|
|
20268
|
+
...super.getStatus(),
|
|
20269
|
+
projectId: this.projectId,
|
|
20270
|
+
datasetId: this.datasetId,
|
|
20271
|
+
resources: this.resources,
|
|
20272
|
+
logTable: this.logTable
|
|
20273
|
+
};
|
|
20274
|
+
}
|
|
20275
|
+
}
|
|
20276
|
+
|
|
20277
|
+
class PostgresReplicator extends BaseReplicator {
|
|
20278
|
+
constructor(config = {}, resources = {}) {
|
|
20279
|
+
super(config);
|
|
20280
|
+
this.connectionString = config.connectionString;
|
|
20281
|
+
this.host = config.host;
|
|
20282
|
+
this.port = config.port || 5432;
|
|
20283
|
+
this.database = config.database;
|
|
20284
|
+
this.user = config.user;
|
|
20285
|
+
this.password = config.password;
|
|
20286
|
+
this.client = null;
|
|
20287
|
+
this.ssl = config.ssl;
|
|
20288
|
+
this.logTable = config.logTable;
|
|
20289
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
20290
|
+
}
|
|
20291
|
+
parseResourcesConfig(resources) {
|
|
20292
|
+
const parsed = {};
|
|
20293
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
20294
|
+
if (typeof config === "string") {
|
|
20295
|
+
parsed[resourceName] = [{
|
|
20296
|
+
table: config,
|
|
20297
|
+
actions: ["insert"]
|
|
20298
|
+
}];
|
|
20299
|
+
} else if (Array.isArray(config)) {
|
|
20300
|
+
parsed[resourceName] = config.map((item) => {
|
|
20301
|
+
if (typeof item === "string") {
|
|
20302
|
+
return { table: item, actions: ["insert"] };
|
|
20303
|
+
}
|
|
20304
|
+
return {
|
|
20305
|
+
table: item.table,
|
|
20306
|
+
actions: item.actions || ["insert"]
|
|
20307
|
+
};
|
|
20308
|
+
});
|
|
20309
|
+
} else if (typeof config === "object") {
|
|
20310
|
+
parsed[resourceName] = [{
|
|
20311
|
+
table: config.table,
|
|
20312
|
+
actions: config.actions || ["insert"]
|
|
20313
|
+
}];
|
|
20314
|
+
}
|
|
20315
|
+
}
|
|
20316
|
+
return parsed;
|
|
20317
|
+
}
|
|
20318
|
+
validateConfig() {
|
|
20319
|
+
const errors = [];
|
|
20320
|
+
if (!this.connectionString && (!this.host || !this.database)) {
|
|
20321
|
+
errors.push("Either connectionString or host+database must be provided");
|
|
20322
|
+
}
|
|
20323
|
+
if (Object.keys(this.resources).length === 0) {
|
|
20324
|
+
errors.push("At least one resource must be configured");
|
|
20325
|
+
}
|
|
20326
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
20327
|
+
for (const tableConfig of tables) {
|
|
20328
|
+
if (!tableConfig.table) {
|
|
20329
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
20330
|
+
}
|
|
20331
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
20332
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
20333
|
+
}
|
|
20334
|
+
const validActions = ["insert", "update", "delete"];
|
|
20335
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
20336
|
+
if (invalidActions.length > 0) {
|
|
20337
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
20338
|
+
}
|
|
20339
|
+
}
|
|
20340
|
+
}
|
|
20341
|
+
return { isValid: errors.length === 0, errors };
|
|
20342
|
+
}
|
|
20343
|
+
async initialize(database) {
|
|
20344
|
+
await super.initialize(database);
|
|
20345
|
+
try {
|
|
20346
|
+
const { Client } = await import('pg');
|
|
20347
|
+
const config = this.connectionString ? {
|
|
20348
|
+
connectionString: this.connectionString,
|
|
20349
|
+
ssl: this.ssl
|
|
20350
|
+
} : {
|
|
20351
|
+
host: this.host,
|
|
20352
|
+
port: this.port,
|
|
20353
|
+
database: this.database,
|
|
20354
|
+
user: this.user,
|
|
20355
|
+
password: this.password,
|
|
20356
|
+
ssl: this.ssl
|
|
20357
|
+
};
|
|
20358
|
+
this.client = new Client(config);
|
|
20359
|
+
await this.client.connect();
|
|
20360
|
+
if (this.logTable) {
|
|
20361
|
+
await this.createLogTableIfNotExists();
|
|
20362
|
+
}
|
|
20363
|
+
this.emit("initialized", {
|
|
20364
|
+
replicator: this.name,
|
|
20365
|
+
database: this.database || "postgres",
|
|
20366
|
+
resources: Object.keys(this.resources)
|
|
20367
|
+
});
|
|
20368
|
+
} catch (error) {
|
|
20369
|
+
this.emit("initialization_error", {
|
|
20370
|
+
replicator: this.name,
|
|
20371
|
+
error: error.message
|
|
20372
|
+
});
|
|
20373
|
+
throw error;
|
|
20374
|
+
}
|
|
20375
|
+
}
|
|
20376
|
+
async createLogTableIfNotExists() {
|
|
20377
|
+
const createTableQuery = `
|
|
20378
|
+
CREATE TABLE IF NOT EXISTS ${this.logTable} (
|
|
20379
|
+
id SERIAL PRIMARY KEY,
|
|
20380
|
+
resource_name VARCHAR(255) NOT NULL,
|
|
20381
|
+
operation VARCHAR(50) NOT NULL,
|
|
20382
|
+
record_id VARCHAR(255) NOT NULL,
|
|
20383
|
+
data JSONB,
|
|
20384
|
+
timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
20385
|
+
source VARCHAR(100) DEFAULT 's3db-replication',
|
|
20386
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
20387
|
+
);
|
|
20388
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_resource_name ON ${this.logTable}(resource_name);
|
|
20389
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_operation ON ${this.logTable}(operation);
|
|
20390
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_record_id ON ${this.logTable}(record_id);
|
|
20391
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_timestamp ON ${this.logTable}(timestamp);
|
|
20392
|
+
`;
|
|
20393
|
+
await this.client.query(createTableQuery);
|
|
20394
|
+
}
|
|
20395
|
+
shouldReplicateResource(resourceName) {
|
|
20396
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
20397
|
+
}
|
|
20398
|
+
shouldReplicateAction(resourceName, operation) {
|
|
20399
|
+
if (!this.resources[resourceName]) return false;
|
|
20400
|
+
return this.resources[resourceName].some(
|
|
20401
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
20402
|
+
);
|
|
20403
|
+
}
|
|
20404
|
+
getTablesForResource(resourceName, operation) {
|
|
20405
|
+
if (!this.resources[resourceName]) return [];
|
|
20406
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => tableConfig.table);
|
|
20407
|
+
}
|
|
20408
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
20409
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
20410
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
20411
|
+
}
|
|
20412
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
20413
|
+
return { skipped: true, reason: "action_not_included" };
|
|
20414
|
+
}
|
|
20415
|
+
const tables = this.getTablesForResource(resourceName, operation);
|
|
20416
|
+
if (tables.length === 0) {
|
|
20417
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
20418
|
+
}
|
|
20419
|
+
const results = [];
|
|
20420
|
+
const errors = [];
|
|
20421
|
+
try {
|
|
20422
|
+
for (const table of tables) {
|
|
20423
|
+
try {
|
|
20424
|
+
let result;
|
|
20425
|
+
if (operation === "insert") {
|
|
20426
|
+
const keys = Object.keys(data);
|
|
20427
|
+
const values = keys.map((k) => data[k]);
|
|
20428
|
+
const columns = keys.map((k) => `"${k}"`).join(", ");
|
|
20429
|
+
const params = keys.map((_, i) => `$${i + 1}`).join(", ");
|
|
20430
|
+
const sql = `INSERT INTO ${table} (${columns}) VALUES (${params}) ON CONFLICT (id) DO NOTHING RETURNING *`;
|
|
20431
|
+
result = await this.client.query(sql, values);
|
|
20432
|
+
} else if (operation === "update") {
|
|
20433
|
+
const keys = Object.keys(data).filter((k) => k !== "id");
|
|
20434
|
+
const setClause = keys.map((k, i) => `"${k}"=$${i + 1}`).join(", ");
|
|
20435
|
+
const values = keys.map((k) => data[k]);
|
|
20436
|
+
values.push(id);
|
|
20437
|
+
const sql = `UPDATE ${table} SET ${setClause} WHERE id=$${keys.length + 1} RETURNING *`;
|
|
20438
|
+
result = await this.client.query(sql, values);
|
|
20439
|
+
} else if (operation === "delete") {
|
|
20440
|
+
const sql = `DELETE FROM ${table} WHERE id=$1 RETURNING *`;
|
|
20441
|
+
result = await this.client.query(sql, [id]);
|
|
20442
|
+
} else {
|
|
20443
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
20444
|
+
}
|
|
20445
|
+
results.push({
|
|
20446
|
+
table,
|
|
20447
|
+
success: true,
|
|
20448
|
+
rows: result.rows,
|
|
20449
|
+
rowCount: result.rowCount
|
|
20450
|
+
});
|
|
20451
|
+
} catch (error) {
|
|
20452
|
+
errors.push({
|
|
20453
|
+
table,
|
|
20454
|
+
error: error.message
|
|
20455
|
+
});
|
|
20456
|
+
}
|
|
20457
|
+
}
|
|
20458
|
+
if (this.logTable) {
|
|
20459
|
+
try {
|
|
20460
|
+
await this.client.query(
|
|
20461
|
+
`INSERT INTO ${this.logTable} (resource_name, operation, record_id, data, timestamp, source) VALUES ($1, $2, $3, $4, $5, $6)`,
|
|
20462
|
+
[resourceName, operation, id, JSON.stringify(data), (/* @__PURE__ */ new Date()).toISOString(), "s3db-replication"]
|
|
20463
|
+
);
|
|
20464
|
+
} catch (error) {
|
|
20465
|
+
console.warn(`Failed to log operation to ${this.logTable}:`, error.message);
|
|
20466
|
+
}
|
|
20467
|
+
}
|
|
20468
|
+
const success = errors.length === 0;
|
|
20469
|
+
this.emit("replicated", {
|
|
20470
|
+
replicator: this.name,
|
|
20471
|
+
resourceName,
|
|
20472
|
+
operation,
|
|
20473
|
+
id,
|
|
20474
|
+
tables,
|
|
20475
|
+
results,
|
|
20476
|
+
errors,
|
|
20477
|
+
success
|
|
20478
|
+
});
|
|
20479
|
+
return {
|
|
20480
|
+
success,
|
|
20481
|
+
results,
|
|
20482
|
+
errors,
|
|
20483
|
+
tables
|
|
20484
|
+
};
|
|
20485
|
+
} catch (error) {
|
|
20486
|
+
this.emit("replication_error", {
|
|
20487
|
+
replicator: this.name,
|
|
20488
|
+
resourceName,
|
|
20489
|
+
operation,
|
|
20490
|
+
id,
|
|
20491
|
+
error: error.message
|
|
20492
|
+
});
|
|
20493
|
+
return { success: false, error: error.message };
|
|
20494
|
+
}
|
|
20495
|
+
}
|
|
20496
|
+
async replicateBatch(resourceName, records) {
|
|
20497
|
+
const results = [];
|
|
20498
|
+
const errors = [];
|
|
20499
|
+
for (const record of records) {
|
|
17901
20500
|
try {
|
|
17902
|
-
const
|
|
17903
|
-
|
|
20501
|
+
const res = await this.replicate(
|
|
20502
|
+
resourceName,
|
|
20503
|
+
record.operation,
|
|
20504
|
+
record.data,
|
|
20505
|
+
record.id,
|
|
20506
|
+
record.beforeData
|
|
20507
|
+
);
|
|
20508
|
+
results.push(res);
|
|
17904
20509
|
} catch (err) {
|
|
17905
|
-
|
|
20510
|
+
errors.push({ id: record.id, error: err.message });
|
|
17906
20511
|
}
|
|
17907
|
-
|
|
17908
|
-
|
|
17909
|
-
|
|
20512
|
+
}
|
|
20513
|
+
return {
|
|
20514
|
+
success: errors.length === 0,
|
|
20515
|
+
results,
|
|
20516
|
+
errors
|
|
17910
20517
|
};
|
|
17911
|
-
|
|
17912
|
-
|
|
17913
|
-
|
|
17914
|
-
|
|
17915
|
-
|
|
17916
|
-
|
|
17917
|
-
|
|
17918
|
-
|
|
20518
|
+
}
|
|
20519
|
+
async testConnection() {
|
|
20520
|
+
try {
|
|
20521
|
+
if (!this.client) await this.initialize();
|
|
20522
|
+
await this.client.query("SELECT 1");
|
|
20523
|
+
return true;
|
|
20524
|
+
} catch (error) {
|
|
20525
|
+
this.emit("connection_error", { replicator: this.name, error: error.message });
|
|
20526
|
+
return false;
|
|
20527
|
+
}
|
|
20528
|
+
}
|
|
20529
|
+
async cleanup() {
|
|
20530
|
+
if (this.client) await this.client.end();
|
|
20531
|
+
}
|
|
20532
|
+
getStatus() {
|
|
20533
|
+
return {
|
|
20534
|
+
...super.getStatus(),
|
|
20535
|
+
database: this.database || "postgres",
|
|
20536
|
+
resources: this.resources,
|
|
20537
|
+
logTable: this.logTable
|
|
17919
20538
|
};
|
|
17920
|
-
|
|
17921
|
-
|
|
17922
|
-
|
|
17923
|
-
|
|
20539
|
+
}
|
|
20540
|
+
}
|
|
20541
|
+
|
|
20542
|
+
const REPLICATOR_DRIVERS = {
|
|
20543
|
+
s3db: S3dbReplicator,
|
|
20544
|
+
sqs: SqsReplicator,
|
|
20545
|
+
bigquery: BigqueryReplicator,
|
|
20546
|
+
postgres: PostgresReplicator
|
|
20547
|
+
};
|
|
20548
|
+
function createReplicator(driver, config = {}, resources = []) {
|
|
20549
|
+
const ReplicatorClass = REPLICATOR_DRIVERS[driver];
|
|
20550
|
+
if (!ReplicatorClass) {
|
|
20551
|
+
throw new Error(`Unknown replicator driver: ${driver}. Available drivers: ${Object.keys(REPLICATOR_DRIVERS).join(", ")}`);
|
|
20552
|
+
}
|
|
20553
|
+
return new ReplicatorClass(config, resources);
|
|
20554
|
+
}
|
|
20555
|
+
function validateReplicatorConfig(driver, config, resources = []) {
|
|
20556
|
+
const replicator = createReplicator(driver, config, resources);
|
|
20557
|
+
return replicator.validateConfig();
|
|
20558
|
+
}
|
|
20559
|
+
|
|
20560
|
+
const gzipAsync = promisify(gzip);
|
|
20561
|
+
const gunzipAsync = promisify(gunzip);
|
|
20562
|
+
class ReplicationPlugin extends Plugin {
|
|
20563
|
+
constructor(options = {}) {
|
|
20564
|
+
super();
|
|
20565
|
+
this.config = {
|
|
20566
|
+
enabled: options.enabled !== false,
|
|
20567
|
+
replicators: options.replicators || [],
|
|
20568
|
+
syncMode: options.syncMode || "async",
|
|
20569
|
+
// 'sync' or 'async'
|
|
20570
|
+
retryAttempts: options.retryAttempts || 3,
|
|
20571
|
+
retryDelay: options.retryDelay || 1e3,
|
|
20572
|
+
// ms
|
|
20573
|
+
batchSize: options.batchSize || 10,
|
|
20574
|
+
compression: options.compression || false,
|
|
20575
|
+
// Enable compression
|
|
20576
|
+
compressionLevel: options.compressionLevel || 6,
|
|
20577
|
+
// 0-9
|
|
20578
|
+
...options
|
|
17924
20579
|
};
|
|
17925
|
-
|
|
17926
|
-
|
|
17927
|
-
|
|
20580
|
+
this.replicators = [];
|
|
20581
|
+
this.queue = [];
|
|
20582
|
+
this.isProcessing = false;
|
|
20583
|
+
this.stats = {
|
|
20584
|
+
totalOperations: 0,
|
|
20585
|
+
successfulOperations: 0,
|
|
20586
|
+
failedOperations: 0,
|
|
20587
|
+
lastSync: null
|
|
20588
|
+
};
|
|
20589
|
+
}
|
|
20590
|
+
/**
|
|
20591
|
+
* Process data according to replication mode
|
|
20592
|
+
*/
|
|
20593
|
+
processDataForReplication(data, metadata = {}) {
|
|
20594
|
+
switch (this.config.replicationMode) {
|
|
20595
|
+
case "exact-copy":
|
|
20596
|
+
return {
|
|
20597
|
+
body: data,
|
|
20598
|
+
metadata
|
|
20599
|
+
};
|
|
20600
|
+
case "just-metadata":
|
|
20601
|
+
return {
|
|
20602
|
+
body: null,
|
|
20603
|
+
metadata
|
|
20604
|
+
};
|
|
20605
|
+
case "all-in-body":
|
|
20606
|
+
return {
|
|
20607
|
+
body: {
|
|
20608
|
+
data,
|
|
20609
|
+
metadata,
|
|
20610
|
+
replicationMode: this.config.replicationMode,
|
|
20611
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
20612
|
+
},
|
|
20613
|
+
metadata: {
|
|
20614
|
+
replicationMode: this.config.replicationMode,
|
|
20615
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
20616
|
+
}
|
|
20617
|
+
};
|
|
20618
|
+
default:
|
|
20619
|
+
return {
|
|
20620
|
+
body: data,
|
|
20621
|
+
metadata
|
|
20622
|
+
};
|
|
20623
|
+
}
|
|
20624
|
+
}
|
|
20625
|
+
/**
|
|
20626
|
+
* Compress data if compression is enabled
|
|
20627
|
+
*/
|
|
20628
|
+
async compressData(data) {
|
|
20629
|
+
if (!this.config.compression || !data) {
|
|
20630
|
+
return data;
|
|
20631
|
+
}
|
|
20632
|
+
try {
|
|
20633
|
+
const jsonString = JSON.stringify(data);
|
|
20634
|
+
const compressed = await gzipAsync(jsonString, { level: this.config.compressionLevel });
|
|
20635
|
+
return compressed.toString("base64");
|
|
20636
|
+
} catch (error) {
|
|
20637
|
+
this.emit("replication.compression.failed", { error, data });
|
|
20638
|
+
return data;
|
|
20639
|
+
}
|
|
20640
|
+
}
|
|
20641
|
+
/**
|
|
20642
|
+
* Decompress data if it was compressed
|
|
20643
|
+
*/
|
|
20644
|
+
async decompressData(data) {
|
|
20645
|
+
if (!this.config.compression || !data) {
|
|
20646
|
+
return data;
|
|
20647
|
+
}
|
|
20648
|
+
try {
|
|
20649
|
+
if (typeof data === "string" && data.startsWith("H4sI")) {
|
|
20650
|
+
const buffer = Buffer.from(data, "base64");
|
|
20651
|
+
const decompressed = await gunzipAsync(buffer);
|
|
20652
|
+
return JSON.parse(decompressed.toString());
|
|
20653
|
+
}
|
|
20654
|
+
return data;
|
|
20655
|
+
} catch (error) {
|
|
20656
|
+
this.emit("replication.decompression.failed", { error, data });
|
|
17928
20657
|
return data;
|
|
20658
|
+
}
|
|
20659
|
+
}
|
|
20660
|
+
async setup(database) {
|
|
20661
|
+
this.database = database;
|
|
20662
|
+
if (!this.config.enabled) {
|
|
20663
|
+
return;
|
|
20664
|
+
}
|
|
20665
|
+
if (this.config.replicators && this.config.replicators.length > 0) {
|
|
20666
|
+
await this.initializeReplicators();
|
|
20667
|
+
}
|
|
20668
|
+
if (!database.resources.replication_logs) {
|
|
20669
|
+
this.replicationLog = await database.createResource({
|
|
20670
|
+
name: "replication_logs",
|
|
20671
|
+
attributes: {
|
|
20672
|
+
id: "string|required",
|
|
20673
|
+
resourceName: "string|required",
|
|
20674
|
+
operation: "string|required",
|
|
20675
|
+
recordId: "string|required",
|
|
20676
|
+
replicatorId: "string|required",
|
|
20677
|
+
status: "string|required",
|
|
20678
|
+
attempts: "number|required",
|
|
20679
|
+
lastAttempt: "string|required",
|
|
20680
|
+
error: "string|required",
|
|
20681
|
+
data: "object|required",
|
|
20682
|
+
timestamp: "string|required"
|
|
20683
|
+
}
|
|
20684
|
+
});
|
|
20685
|
+
} else {
|
|
20686
|
+
this.replicationLog = database.resources.replication_logs;
|
|
20687
|
+
}
|
|
20688
|
+
for (const resourceName in database.resources) {
|
|
20689
|
+
if (resourceName !== "replication_logs") {
|
|
20690
|
+
this.installHooks(database.resources[resourceName]);
|
|
20691
|
+
}
|
|
20692
|
+
}
|
|
20693
|
+
const originalCreateResource = database.createResource.bind(database);
|
|
20694
|
+
database.createResource = async (config) => {
|
|
20695
|
+
const resource = await originalCreateResource(config);
|
|
20696
|
+
if (resource && resource.name !== "replication_logs") {
|
|
20697
|
+
this.installHooks(resource);
|
|
20698
|
+
}
|
|
20699
|
+
return resource;
|
|
17929
20700
|
};
|
|
17930
|
-
|
|
17931
|
-
|
|
17932
|
-
|
|
20701
|
+
this.startQueueProcessor();
|
|
20702
|
+
}
|
|
20703
|
+
async initializeReplicators() {
|
|
20704
|
+
for (const replicatorConfig of this.config.replicators) {
|
|
20705
|
+
try {
|
|
20706
|
+
const { driver, config: replicatorConfigData, resources = [] } = replicatorConfig;
|
|
20707
|
+
const validation = validateReplicatorConfig(driver, replicatorConfigData, resources);
|
|
20708
|
+
if (!validation.isValid) {
|
|
20709
|
+
this.emit("replicator.validation.failed", {
|
|
20710
|
+
driver,
|
|
20711
|
+
errors: validation.errors
|
|
20712
|
+
});
|
|
20713
|
+
continue;
|
|
20714
|
+
}
|
|
20715
|
+
const replicator = createReplicator(driver, replicatorConfigData, resources);
|
|
20716
|
+
await replicator.initialize(this.database);
|
|
20717
|
+
replicator.on("replicated", (data) => {
|
|
20718
|
+
this.emit("replication.success", data);
|
|
20719
|
+
});
|
|
20720
|
+
replicator.on("replication_error", (data) => {
|
|
20721
|
+
this.emit("replication.failed", data);
|
|
20722
|
+
});
|
|
20723
|
+
this.replicators.push({
|
|
20724
|
+
id: `${driver}-${Date.now()}`,
|
|
20725
|
+
driver,
|
|
20726
|
+
config: replicatorConfigData,
|
|
20727
|
+
resources,
|
|
20728
|
+
instance: replicator
|
|
20729
|
+
});
|
|
20730
|
+
this.emit("replicator.initialized", {
|
|
20731
|
+
driver,
|
|
20732
|
+
config: replicatorConfigData,
|
|
20733
|
+
resources
|
|
20734
|
+
});
|
|
20735
|
+
} catch (error) {
|
|
20736
|
+
this.emit("replicator.initialization.failed", {
|
|
20737
|
+
driver: replicatorConfig.driver,
|
|
20738
|
+
error: error.message
|
|
20739
|
+
});
|
|
20740
|
+
}
|
|
20741
|
+
}
|
|
20742
|
+
}
|
|
20743
|
+
async start() {
|
|
20744
|
+
}
|
|
20745
|
+
async stop() {
|
|
20746
|
+
this.isProcessing = false;
|
|
20747
|
+
await this.processQueue();
|
|
20748
|
+
}
|
|
20749
|
+
installHooks(resource) {
|
|
20750
|
+
if (!resource || resource.name === "replication_logs") return;
|
|
20751
|
+
const originalDataMap = /* @__PURE__ */ new Map();
|
|
20752
|
+
resource.addHook("afterInsert", async (data) => {
|
|
20753
|
+
await this.queueReplication(resource.name, "insert", data.id, data);
|
|
20754
|
+
return data;
|
|
20755
|
+
});
|
|
20756
|
+
resource.addHook("preUpdate", async (data) => {
|
|
20757
|
+
if (data.id) {
|
|
20758
|
+
try {
|
|
20759
|
+
const originalData = await resource.get(data.id);
|
|
20760
|
+
originalDataMap.set(data.id, originalData);
|
|
20761
|
+
} catch (error) {
|
|
20762
|
+
originalDataMap.set(data.id, { id: data.id });
|
|
20763
|
+
}
|
|
20764
|
+
}
|
|
20765
|
+
return data;
|
|
20766
|
+
});
|
|
20767
|
+
resource.addHook("afterUpdate", async (data) => {
|
|
20768
|
+
const beforeData = originalDataMap.get(data.id);
|
|
20769
|
+
await this.queueReplication(resource.name, "update", data.id, data, beforeData);
|
|
20770
|
+
originalDataMap.delete(data.id);
|
|
17933
20771
|
return data;
|
|
20772
|
+
});
|
|
20773
|
+
resource.addHook("afterDelete", async (data) => {
|
|
20774
|
+
await this.queueReplication(resource.name, "delete", data.id, data);
|
|
20775
|
+
return data;
|
|
20776
|
+
});
|
|
20777
|
+
const originalDeleteMany = resource.deleteMany.bind(resource);
|
|
20778
|
+
resource.deleteMany = async (ids) => {
|
|
20779
|
+
const result = await originalDeleteMany(ids);
|
|
20780
|
+
if (result && result.length > 0) {
|
|
20781
|
+
for (const id of ids) {
|
|
20782
|
+
await this.queueReplication(resource.name, "delete", id, { id });
|
|
20783
|
+
}
|
|
20784
|
+
}
|
|
20785
|
+
return result;
|
|
20786
|
+
};
|
|
20787
|
+
}
|
|
20788
|
+
async queueReplication(resourceName, operation, recordId, data, beforeData = null) {
|
|
20789
|
+
if (!this.config.enabled) {
|
|
20790
|
+
return;
|
|
20791
|
+
}
|
|
20792
|
+
if (this.replicators.length === 0) {
|
|
20793
|
+
return;
|
|
20794
|
+
}
|
|
20795
|
+
const applicableReplicators = this.replicators.filter(
|
|
20796
|
+
(replicator) => replicator.instance.shouldReplicateResource(resourceName)
|
|
20797
|
+
);
|
|
20798
|
+
if (applicableReplicators.length === 0) {
|
|
20799
|
+
return;
|
|
20800
|
+
}
|
|
20801
|
+
const item = {
|
|
20802
|
+
id: `repl-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
20803
|
+
resourceName,
|
|
20804
|
+
operation,
|
|
20805
|
+
recordId,
|
|
20806
|
+
data: lodashEs.isPlainObject(data) ? data : { raw: data },
|
|
20807
|
+
beforeData: beforeData ? lodashEs.isPlainObject(beforeData) ? beforeData : { raw: beforeData } : null,
|
|
20808
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20809
|
+
attempts: 0
|
|
20810
|
+
};
|
|
20811
|
+
const logId = await this.logReplication(item);
|
|
20812
|
+
if (this.config.syncMode === "sync") {
|
|
20813
|
+
try {
|
|
20814
|
+
const result = await this.processReplicationItem(item);
|
|
20815
|
+
if (logId) {
|
|
20816
|
+
await this.updateReplicationLog(logId, {
|
|
20817
|
+
status: result.success ? "success" : "failed",
|
|
20818
|
+
attempts: 1,
|
|
20819
|
+
error: result.success ? "" : JSON.stringify(result.results)
|
|
20820
|
+
});
|
|
20821
|
+
}
|
|
20822
|
+
this.stats.totalOperations++;
|
|
20823
|
+
if (result.success) {
|
|
20824
|
+
this.stats.successfulOperations++;
|
|
20825
|
+
} else {
|
|
20826
|
+
this.stats.failedOperations++;
|
|
20827
|
+
}
|
|
20828
|
+
} catch (error) {
|
|
20829
|
+
if (logId) {
|
|
20830
|
+
await this.updateReplicationLog(logId, {
|
|
20831
|
+
status: "failed",
|
|
20832
|
+
attempts: 1,
|
|
20833
|
+
error: error.message
|
|
20834
|
+
});
|
|
20835
|
+
}
|
|
20836
|
+
this.stats.failedOperations++;
|
|
20837
|
+
}
|
|
20838
|
+
} else {
|
|
20839
|
+
this.queue.push(item);
|
|
20840
|
+
this.emit("replication.queued", { item, queueLength: this.queue.length });
|
|
20841
|
+
}
|
|
20842
|
+
}
|
|
20843
|
+
async processReplicationItem(item) {
|
|
20844
|
+
const { resourceName, operation, recordId, data, beforeData } = item;
|
|
20845
|
+
const applicableReplicators = this.replicators.filter(
|
|
20846
|
+
(replicator) => replicator.instance.shouldReplicateResource(resourceName)
|
|
20847
|
+
);
|
|
20848
|
+
if (applicableReplicators.length === 0) {
|
|
20849
|
+
return { success: true, skipped: true, reason: "no_applicable_replicators" };
|
|
20850
|
+
}
|
|
20851
|
+
const results = [];
|
|
20852
|
+
for (const replicator of applicableReplicators) {
|
|
20853
|
+
try {
|
|
20854
|
+
const result = await replicator.instance.replicate(resourceName, operation, data, recordId, beforeData);
|
|
20855
|
+
results.push({
|
|
20856
|
+
replicatorId: replicator.id,
|
|
20857
|
+
driver: replicator.driver,
|
|
20858
|
+
success: result.success,
|
|
20859
|
+
error: result.error,
|
|
20860
|
+
skipped: result.skipped
|
|
20861
|
+
});
|
|
20862
|
+
} catch (error) {
|
|
20863
|
+
results.push({
|
|
20864
|
+
replicatorId: replicator.id,
|
|
20865
|
+
driver: replicator.driver,
|
|
20866
|
+
success: false,
|
|
20867
|
+
error: error.message
|
|
20868
|
+
});
|
|
20869
|
+
}
|
|
20870
|
+
}
|
|
20871
|
+
return {
|
|
20872
|
+
success: results.every((r) => r.success || r.skipped),
|
|
20873
|
+
results
|
|
20874
|
+
};
|
|
20875
|
+
}
|
|
20876
|
+
async logReplication(item) {
|
|
20877
|
+
if (!this.replicationLog) return;
|
|
20878
|
+
try {
|
|
20879
|
+
const logId = `log-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
20880
|
+
await this.replicationLog.insert({
|
|
20881
|
+
id: logId,
|
|
20882
|
+
resourceName: item.resourceName,
|
|
20883
|
+
operation: item.operation,
|
|
20884
|
+
recordId: item.recordId,
|
|
20885
|
+
replicatorId: "all",
|
|
20886
|
+
// Will be updated with specific replicator results
|
|
20887
|
+
status: "queued",
|
|
20888
|
+
attempts: 0,
|
|
20889
|
+
lastAttempt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20890
|
+
error: "",
|
|
20891
|
+
data: lodashEs.isPlainObject(item.data) ? item.data : { raw: item.data },
|
|
20892
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
20893
|
+
});
|
|
20894
|
+
return logId;
|
|
20895
|
+
} catch (error) {
|
|
20896
|
+
this.emit("replication.log.failed", { error: error.message, item });
|
|
20897
|
+
return null;
|
|
20898
|
+
}
|
|
20899
|
+
}
|
|
20900
|
+
async updateReplicationLog(logId, updates) {
|
|
20901
|
+
if (!this.replicationLog) return;
|
|
20902
|
+
try {
|
|
20903
|
+
await this.replicationLog.update(logId, {
|
|
20904
|
+
...updates,
|
|
20905
|
+
lastAttempt: (/* @__PURE__ */ new Date()).toISOString()
|
|
20906
|
+
});
|
|
20907
|
+
} catch (error) {
|
|
20908
|
+
this.emit("replication.updateLog.failed", { error: error.message, logId, updates });
|
|
20909
|
+
}
|
|
20910
|
+
}
|
|
20911
|
+
startQueueProcessor() {
|
|
20912
|
+
if (this.isProcessing) return;
|
|
20913
|
+
this.isProcessing = true;
|
|
20914
|
+
this.processQueueLoop();
|
|
20915
|
+
}
|
|
20916
|
+
async processQueueLoop() {
|
|
20917
|
+
while (this.isProcessing) {
|
|
20918
|
+
if (this.queue.length > 0) {
|
|
20919
|
+
const batch = this.queue.splice(0, this.config.batchSize);
|
|
20920
|
+
for (const item of batch) {
|
|
20921
|
+
await this.processReplicationItem(item);
|
|
20922
|
+
}
|
|
20923
|
+
} else {
|
|
20924
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
20925
|
+
}
|
|
20926
|
+
}
|
|
20927
|
+
}
|
|
20928
|
+
async processQueue() {
|
|
20929
|
+
if (this.queue.length === 0) return;
|
|
20930
|
+
const item = this.queue.shift();
|
|
20931
|
+
let attempts = 0;
|
|
20932
|
+
let lastError = null;
|
|
20933
|
+
while (attempts < this.config.retryAttempts) {
|
|
20934
|
+
try {
|
|
20935
|
+
attempts++;
|
|
20936
|
+
this.emit("replication.retry.started", {
|
|
20937
|
+
item,
|
|
20938
|
+
attempt: attempts,
|
|
20939
|
+
maxAttempts: this.config.retryAttempts
|
|
20940
|
+
});
|
|
20941
|
+
const result = await this.processReplicationItem(item);
|
|
20942
|
+
if (result.success) {
|
|
20943
|
+
this.stats.successfulOperations++;
|
|
20944
|
+
this.emit("replication.success", {
|
|
20945
|
+
item,
|
|
20946
|
+
attempts,
|
|
20947
|
+
results: result.results,
|
|
20948
|
+
stats: this.stats
|
|
20949
|
+
});
|
|
20950
|
+
return;
|
|
20951
|
+
} else {
|
|
20952
|
+
lastError = result.results;
|
|
20953
|
+
if (attempts < this.config.retryAttempts) {
|
|
20954
|
+
await new Promise((resolve) => setTimeout(resolve, this.config.retryDelay * attempts));
|
|
20955
|
+
}
|
|
20956
|
+
}
|
|
20957
|
+
} catch (error) {
|
|
20958
|
+
lastError = error.message;
|
|
20959
|
+
if (attempts < this.config.retryAttempts) {
|
|
20960
|
+
await new Promise((resolve) => setTimeout(resolve, this.config.retryDelay * attempts));
|
|
20961
|
+
} else {
|
|
20962
|
+
this.emit("replication.retry.exhausted", {
|
|
20963
|
+
attempts,
|
|
20964
|
+
lastError,
|
|
20965
|
+
item
|
|
20966
|
+
});
|
|
20967
|
+
}
|
|
20968
|
+
}
|
|
20969
|
+
}
|
|
20970
|
+
this.stats.failedOperations++;
|
|
20971
|
+
this.emit("replication.failed", {
|
|
20972
|
+
attempts,
|
|
20973
|
+
lastError,
|
|
20974
|
+
item,
|
|
20975
|
+
stats: this.stats
|
|
20976
|
+
});
|
|
20977
|
+
}
|
|
20978
|
+
// Utility methods
|
|
20979
|
+
async getReplicationStats() {
|
|
20980
|
+
const replicatorStats = await Promise.all(
|
|
20981
|
+
this.replicators.map(async (replicator) => {
|
|
20982
|
+
const status = await replicator.instance.getStatus();
|
|
20983
|
+
return {
|
|
20984
|
+
id: replicator.id,
|
|
20985
|
+
driver: replicator.driver,
|
|
20986
|
+
config: replicator.config,
|
|
20987
|
+
status
|
|
20988
|
+
};
|
|
20989
|
+
})
|
|
20990
|
+
);
|
|
20991
|
+
return {
|
|
20992
|
+
enabled: this.config.enabled,
|
|
20993
|
+
replicators: replicatorStats,
|
|
20994
|
+
queue: {
|
|
20995
|
+
length: this.queue.length,
|
|
20996
|
+
isProcessing: this.isProcessing
|
|
20997
|
+
},
|
|
20998
|
+
stats: this.stats,
|
|
20999
|
+
lastSync: this.stats.lastSync
|
|
17934
21000
|
};
|
|
17935
21001
|
}
|
|
21002
|
+
async getReplicationLogs(options = {}) {
|
|
21003
|
+
if (!this.replicationLog) {
|
|
21004
|
+
return [];
|
|
21005
|
+
}
|
|
21006
|
+
const {
|
|
21007
|
+
resourceName,
|
|
21008
|
+
operation,
|
|
21009
|
+
status,
|
|
21010
|
+
limit = 100,
|
|
21011
|
+
offset = 0
|
|
21012
|
+
} = options;
|
|
21013
|
+
let query = {};
|
|
21014
|
+
if (resourceName) {
|
|
21015
|
+
query.resourceName = resourceName;
|
|
21016
|
+
}
|
|
21017
|
+
if (operation) {
|
|
21018
|
+
query.operation = operation;
|
|
21019
|
+
}
|
|
21020
|
+
if (status) {
|
|
21021
|
+
query.status = status;
|
|
21022
|
+
}
|
|
21023
|
+
const logs = await this.replicationLog.list(query);
|
|
21024
|
+
return logs.slice(offset, offset + limit);
|
|
21025
|
+
}
|
|
21026
|
+
async retryFailedReplications() {
|
|
21027
|
+
if (!this.replicationLog) {
|
|
21028
|
+
return { retried: 0 };
|
|
21029
|
+
}
|
|
21030
|
+
const failedLogs = await this.replicationLog.list({
|
|
21031
|
+
status: "failed"
|
|
21032
|
+
});
|
|
21033
|
+
let retried = 0;
|
|
21034
|
+
for (const log of failedLogs) {
|
|
21035
|
+
try {
|
|
21036
|
+
await this.queueReplication(
|
|
21037
|
+
log.resourceName,
|
|
21038
|
+
log.operation,
|
|
21039
|
+
log.recordId,
|
|
21040
|
+
log.data
|
|
21041
|
+
);
|
|
21042
|
+
retried++;
|
|
21043
|
+
} catch (error) {
|
|
21044
|
+
console.error("Failed to retry replication:", error);
|
|
21045
|
+
}
|
|
21046
|
+
}
|
|
21047
|
+
return { retried };
|
|
21048
|
+
}
|
|
21049
|
+
async syncAllData(replicatorId) {
|
|
21050
|
+
const replicator = this.replicators.find((r) => r.id === replicatorId);
|
|
21051
|
+
if (!replicator) {
|
|
21052
|
+
throw new Error(`Replicator not found: ${replicatorId}`);
|
|
21053
|
+
}
|
|
21054
|
+
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
21055
|
+
for (const resourceName in this.database.resources) {
|
|
21056
|
+
if (resourceName === "replication_logs") continue;
|
|
21057
|
+
if (replicator.instance.shouldReplicateResource(resourceName)) {
|
|
21058
|
+
this.emit("replication.sync.resource", { resourceName, replicatorId });
|
|
21059
|
+
const resource = this.database.resources[resourceName];
|
|
21060
|
+
const allRecords = await resource.getAll();
|
|
21061
|
+
for (const record of allRecords) {
|
|
21062
|
+
await replicator.instance.replicate(resourceName, "insert", record, record.id);
|
|
21063
|
+
}
|
|
21064
|
+
}
|
|
21065
|
+
}
|
|
21066
|
+
this.emit("replication.sync.completed", { replicatorId, stats: this.stats });
|
|
21067
|
+
}
|
|
17936
21068
|
}
|
|
17937
21069
|
|
|
17938
21070
|
exports.AVAILABLE_BEHAVIORS = AVAILABLE_BEHAVIORS;
|
|
21071
|
+
exports.AuditPlugin = AuditPlugin;
|
|
17939
21072
|
exports.AuthenticationError = AuthenticationError;
|
|
17940
21073
|
exports.BaseError = BaseError;
|
|
17941
21074
|
exports.Cache = Cache;
|
|
@@ -17948,8 +21081,10 @@ ${validation.errors.join("\n")}`);
|
|
|
17948
21081
|
exports.DatabaseError = DatabaseError;
|
|
17949
21082
|
exports.EncryptionError = EncryptionError;
|
|
17950
21083
|
exports.ErrorMap = ErrorMap;
|
|
21084
|
+
exports.FullTextPlugin = FullTextPlugin;
|
|
17951
21085
|
exports.InvalidResourceItem = InvalidResourceItem;
|
|
17952
21086
|
exports.MemoryCache = MemoryCache;
|
|
21087
|
+
exports.MetricsPlugin = MetricsPlugin;
|
|
17953
21088
|
exports.MissingMetadata = MissingMetadata;
|
|
17954
21089
|
exports.NoSuchBucket = NoSuchBucket;
|
|
17955
21090
|
exports.NoSuchKey = NoSuchKey;
|
|
@@ -17957,6 +21092,7 @@ ${validation.errors.join("\n")}`);
|
|
|
17957
21092
|
exports.PermissionError = PermissionError;
|
|
17958
21093
|
exports.Plugin = Plugin;
|
|
17959
21094
|
exports.PluginObject = PluginObject;
|
|
21095
|
+
exports.ReplicationPlugin = ReplicationPlugin;
|
|
17960
21096
|
exports.Resource = Resource;
|
|
17961
21097
|
exports.ResourceIdsPageReader = ResourceIdsPageReader;
|
|
17962
21098
|
exports.ResourceIdsReader = ResourceIdsReader;
|