s3db.js 5.2.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1142 -612
- package/dist/s3db.cjs.js +3541 -484
- package/dist/s3db.cjs.min.js +1 -31
- package/dist/s3db.d.ts +891 -62
- package/dist/s3db.es.js +3539 -486
- package/dist/s3db.es.min.js +1 -31
- package/dist/s3db.iife.js +3541 -484
- package/dist/s3db.iife.min.js +1 -31
- package/package.json +33 -14
package/dist/s3db.es.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
/* istanbul ignore file */
|
|
2
1
|
import { customAlphabet, urlAlphabet } from 'nanoid';
|
|
3
|
-
import { chunk, merge, isString as isString$1, isEmpty, invert, uniq, cloneDeep, get as get$1, set, isFunction as isFunction$1 } from 'lodash-es';
|
|
2
|
+
import { chunk, merge, isString as isString$1, isEmpty, invert, uniq, cloneDeep, get as get$1, set, isFunction as isFunction$1, isPlainObject } from 'lodash-es';
|
|
4
3
|
import { PromisePool } from '@supercharge/promise-pool';
|
|
5
4
|
import { S3Client, PutObjectCommand, GetObjectCommand, HeadObjectCommand, CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
|
|
6
5
|
import { createHash } from 'crypto';
|
|
@@ -939,15 +938,24 @@ class Client extends EventEmitter {
|
|
|
939
938
|
if (errorClass) return new errorClass(data);
|
|
940
939
|
return error;
|
|
941
940
|
}
|
|
942
|
-
async putObject({ key, metadata, contentType, body, contentEncoding }) {
|
|
941
|
+
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength }) {
|
|
942
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
943
|
+
const stringMetadata = {};
|
|
944
|
+
if (metadata) {
|
|
945
|
+
for (const [k, v] of Object.entries(metadata)) {
|
|
946
|
+
const validKey = String(k).replace(/[^a-zA-Z0-9\-_]/g, "_");
|
|
947
|
+
stringMetadata[validKey] = String(v);
|
|
948
|
+
}
|
|
949
|
+
}
|
|
943
950
|
const options2 = {
|
|
944
951
|
Bucket: this.config.bucket,
|
|
945
|
-
Key:
|
|
946
|
-
Metadata:
|
|
952
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key,
|
|
953
|
+
Metadata: stringMetadata,
|
|
947
954
|
Body: body || Buffer.alloc(0)
|
|
948
955
|
};
|
|
949
956
|
if (contentType !== void 0) options2.ContentType = contentType;
|
|
950
957
|
if (contentEncoding !== void 0) options2.ContentEncoding = contentEncoding;
|
|
958
|
+
if (contentLength !== void 0) options2.ContentLength = contentLength;
|
|
951
959
|
try {
|
|
952
960
|
const response = await this.sendCommand(new PutObjectCommand(options2));
|
|
953
961
|
this.emit("putObject", response, options2);
|
|
@@ -960,9 +968,10 @@ class Client extends EventEmitter {
|
|
|
960
968
|
}
|
|
961
969
|
}
|
|
962
970
|
async getObject(key) {
|
|
971
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
963
972
|
const options2 = {
|
|
964
973
|
Bucket: this.config.bucket,
|
|
965
|
-
Key: path.join(
|
|
974
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
966
975
|
};
|
|
967
976
|
try {
|
|
968
977
|
const response = await this.sendCommand(new GetObjectCommand(options2));
|
|
@@ -976,9 +985,10 @@ class Client extends EventEmitter {
|
|
|
976
985
|
}
|
|
977
986
|
}
|
|
978
987
|
async headObject(key) {
|
|
988
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
979
989
|
const options2 = {
|
|
980
990
|
Bucket: this.config.bucket,
|
|
981
|
-
Key:
|
|
991
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
982
992
|
};
|
|
983
993
|
try {
|
|
984
994
|
const response = await this.sendCommand(new HeadObjectCommand(options2));
|
|
@@ -1020,9 +1030,10 @@ class Client extends EventEmitter {
|
|
|
1020
1030
|
}
|
|
1021
1031
|
}
|
|
1022
1032
|
async deleteObject(key) {
|
|
1033
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
1023
1034
|
const options2 = {
|
|
1024
1035
|
Bucket: this.config.bucket,
|
|
1025
|
-
Key:
|
|
1036
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
1026
1037
|
};
|
|
1027
1038
|
try {
|
|
1028
1039
|
const response = await this.sendCommand(new DeleteObjectCommand(options2));
|
|
@@ -1036,13 +1047,14 @@ class Client extends EventEmitter {
|
|
|
1036
1047
|
}
|
|
1037
1048
|
}
|
|
1038
1049
|
async deleteObjects(keys) {
|
|
1050
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
1039
1051
|
const packages = chunk(keys, 1e3);
|
|
1040
1052
|
const { results, errors } = await PromisePool.for(packages).withConcurrency(this.parallelism).process(async (keys2) => {
|
|
1041
1053
|
const options2 = {
|
|
1042
1054
|
Bucket: this.config.bucket,
|
|
1043
1055
|
Delete: {
|
|
1044
1056
|
Objects: keys2.map((key) => ({
|
|
1045
|
-
Key:
|
|
1057
|
+
Key: keyPrefix ? path.join(keyPrefix, key) : key
|
|
1046
1058
|
}))
|
|
1047
1059
|
}
|
|
1048
1060
|
};
|
|
@@ -1070,12 +1082,13 @@ class Client extends EventEmitter {
|
|
|
1070
1082
|
* @returns {Promise<number>} Number of objects deleted
|
|
1071
1083
|
*/
|
|
1072
1084
|
async deleteAll({ prefix } = {}) {
|
|
1085
|
+
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
1073
1086
|
let continuationToken;
|
|
1074
1087
|
let totalDeleted = 0;
|
|
1075
1088
|
do {
|
|
1076
1089
|
const listCommand = new ListObjectsV2Command({
|
|
1077
1090
|
Bucket: this.config.bucket,
|
|
1078
|
-
Prefix:
|
|
1091
|
+
Prefix: keyPrefix ? path.join(keyPrefix, prefix) : prefix,
|
|
1079
1092
|
ContinuationToken: continuationToken
|
|
1080
1093
|
});
|
|
1081
1094
|
const listResponse = await this.client.send(listCommand);
|
|
@@ -1217,6 +1230,10 @@ class Client extends EventEmitter {
|
|
|
1217
1230
|
prefix,
|
|
1218
1231
|
offset
|
|
1219
1232
|
});
|
|
1233
|
+
if (!continuationToken) {
|
|
1234
|
+
this.emit("getKeysPage", [], params);
|
|
1235
|
+
return [];
|
|
1236
|
+
}
|
|
1220
1237
|
}
|
|
1221
1238
|
while (truncated) {
|
|
1222
1239
|
const options2 = {
|
|
@@ -1229,8 +1246,8 @@ class Client extends EventEmitter {
|
|
|
1229
1246
|
}
|
|
1230
1247
|
truncated = res.IsTruncated || false;
|
|
1231
1248
|
continuationToken = res.NextContinuationToken;
|
|
1232
|
-
if (keys.length
|
|
1233
|
-
keys = keys.
|
|
1249
|
+
if (keys.length >= amount) {
|
|
1250
|
+
keys = keys.slice(0, amount);
|
|
1234
1251
|
break;
|
|
1235
1252
|
}
|
|
1236
1253
|
}
|
|
@@ -6002,6 +6019,16 @@ if (typeof Object.create === 'function'){
|
|
|
6002
6019
|
};
|
|
6003
6020
|
}
|
|
6004
6021
|
|
|
6022
|
+
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
6023
|
+
function getOwnPropertyDescriptors(obj) {
|
|
6024
|
+
var keys = Object.keys(obj);
|
|
6025
|
+
var descriptors = {};
|
|
6026
|
+
for (var i = 0; i < keys.length; i++) {
|
|
6027
|
+
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
6028
|
+
}
|
|
6029
|
+
return descriptors;
|
|
6030
|
+
};
|
|
6031
|
+
|
|
6005
6032
|
var formatRegExp = /%[sdj%]/g;
|
|
6006
6033
|
function format(f) {
|
|
6007
6034
|
if (!isString(f)) {
|
|
@@ -6487,6 +6514,64 @@ function hasOwnProperty(obj, prop) {
|
|
|
6487
6514
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
6488
6515
|
}
|
|
6489
6516
|
|
|
6517
|
+
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
6518
|
+
|
|
6519
|
+
function promisify(original) {
|
|
6520
|
+
if (typeof original !== 'function')
|
|
6521
|
+
throw new TypeError('The "original" argument must be of type Function');
|
|
6522
|
+
|
|
6523
|
+
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
6524
|
+
var fn = original[kCustomPromisifiedSymbol];
|
|
6525
|
+
if (typeof fn !== 'function') {
|
|
6526
|
+
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
6527
|
+
}
|
|
6528
|
+
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
6529
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
6530
|
+
});
|
|
6531
|
+
return fn;
|
|
6532
|
+
}
|
|
6533
|
+
|
|
6534
|
+
function fn() {
|
|
6535
|
+
var promiseResolve, promiseReject;
|
|
6536
|
+
var promise = new Promise(function (resolve, reject) {
|
|
6537
|
+
promiseResolve = resolve;
|
|
6538
|
+
promiseReject = reject;
|
|
6539
|
+
});
|
|
6540
|
+
|
|
6541
|
+
var args = [];
|
|
6542
|
+
for (var i = 0; i < arguments.length; i++) {
|
|
6543
|
+
args.push(arguments[i]);
|
|
6544
|
+
}
|
|
6545
|
+
args.push(function (err, value) {
|
|
6546
|
+
if (err) {
|
|
6547
|
+
promiseReject(err);
|
|
6548
|
+
} else {
|
|
6549
|
+
promiseResolve(value);
|
|
6550
|
+
}
|
|
6551
|
+
});
|
|
6552
|
+
|
|
6553
|
+
try {
|
|
6554
|
+
original.apply(this, args);
|
|
6555
|
+
} catch (err) {
|
|
6556
|
+
promiseReject(err);
|
|
6557
|
+
}
|
|
6558
|
+
|
|
6559
|
+
return promise;
|
|
6560
|
+
}
|
|
6561
|
+
|
|
6562
|
+
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
6563
|
+
|
|
6564
|
+
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
6565
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
6566
|
+
});
|
|
6567
|
+
return Object.defineProperties(
|
|
6568
|
+
fn,
|
|
6569
|
+
getOwnPropertyDescriptors(original)
|
|
6570
|
+
);
|
|
6571
|
+
}
|
|
6572
|
+
|
|
6573
|
+
promisify.custom = kCustomPromisifiedSymbol;
|
|
6574
|
+
|
|
6490
6575
|
function BufferList() {
|
|
6491
6576
|
this.head = null;
|
|
6492
6577
|
this.tail = null;
|
|
@@ -8716,41 +8801,41 @@ function getSizeBreakdown(mappedObject) {
|
|
|
8716
8801
|
}
|
|
8717
8802
|
|
|
8718
8803
|
const S3_METADATA_LIMIT_BYTES = 2048;
|
|
8719
|
-
async function handleInsert$
|
|
8804
|
+
async function handleInsert$4({ resource, data, mappedData }) {
|
|
8720
8805
|
const totalSize = calculateTotalSize(mappedData);
|
|
8721
8806
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8722
8807
|
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
8723
8808
|
}
|
|
8724
8809
|
return { mappedData, body: "" };
|
|
8725
8810
|
}
|
|
8726
|
-
async function handleUpdate$
|
|
8811
|
+
async function handleUpdate$4({ resource, id, data, mappedData }) {
|
|
8727
8812
|
const totalSize = calculateTotalSize(mappedData);
|
|
8728
8813
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8729
8814
|
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
8730
8815
|
}
|
|
8731
8816
|
return { mappedData, body: "" };
|
|
8732
8817
|
}
|
|
8733
|
-
async function handleUpsert$
|
|
8818
|
+
async function handleUpsert$4({ resource, id, data, mappedData }) {
|
|
8734
8819
|
const totalSize = calculateTotalSize(mappedData);
|
|
8735
8820
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8736
8821
|
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
8737
8822
|
}
|
|
8738
8823
|
return { mappedData, body: "" };
|
|
8739
8824
|
}
|
|
8740
|
-
async function handleGet$
|
|
8825
|
+
async function handleGet$4({ resource, metadata, body }) {
|
|
8741
8826
|
return { metadata, body };
|
|
8742
8827
|
}
|
|
8743
8828
|
|
|
8744
8829
|
var enforceLimits = /*#__PURE__*/Object.freeze({
|
|
8745
8830
|
__proto__: null,
|
|
8746
8831
|
S3_METADATA_LIMIT_BYTES: S3_METADATA_LIMIT_BYTES,
|
|
8747
|
-
handleGet: handleGet$
|
|
8748
|
-
handleInsert: handleInsert$
|
|
8749
|
-
handleUpdate: handleUpdate$
|
|
8750
|
-
handleUpsert: handleUpsert$
|
|
8832
|
+
handleGet: handleGet$4,
|
|
8833
|
+
handleInsert: handleInsert$4,
|
|
8834
|
+
handleUpdate: handleUpdate$4,
|
|
8835
|
+
handleUpsert: handleUpsert$4
|
|
8751
8836
|
});
|
|
8752
8837
|
|
|
8753
|
-
async function handleInsert$
|
|
8838
|
+
async function handleInsert$3({ resource, data, mappedData }) {
|
|
8754
8839
|
const totalSize = calculateTotalSize(mappedData);
|
|
8755
8840
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8756
8841
|
resource.emit("exceedsLimit", {
|
|
@@ -8763,7 +8848,7 @@ async function handleInsert$2({ resource, data, mappedData }) {
|
|
|
8763
8848
|
}
|
|
8764
8849
|
return { mappedData, body: "" };
|
|
8765
8850
|
}
|
|
8766
|
-
async function handleUpdate$
|
|
8851
|
+
async function handleUpdate$3({ resource, id, data, mappedData }) {
|
|
8767
8852
|
const totalSize = calculateTotalSize(mappedData);
|
|
8768
8853
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8769
8854
|
resource.emit("exceedsLimit", {
|
|
@@ -8777,7 +8862,7 @@ async function handleUpdate$2({ resource, id, data, mappedData }) {
|
|
|
8777
8862
|
}
|
|
8778
8863
|
return { mappedData, body: "" };
|
|
8779
8864
|
}
|
|
8780
|
-
async function handleUpsert$
|
|
8865
|
+
async function handleUpsert$3({ resource, id, data, mappedData }) {
|
|
8781
8866
|
const totalSize = calculateTotalSize(mappedData);
|
|
8782
8867
|
if (totalSize > S3_METADATA_LIMIT_BYTES) {
|
|
8783
8868
|
resource.emit("exceedsLimit", {
|
|
@@ -8791,30 +8876,30 @@ async function handleUpsert$2({ resource, id, data, mappedData }) {
|
|
|
8791
8876
|
}
|
|
8792
8877
|
return { mappedData, body: "" };
|
|
8793
8878
|
}
|
|
8794
|
-
async function handleGet$
|
|
8879
|
+
async function handleGet$3({ resource, metadata, body }) {
|
|
8795
8880
|
return { metadata, body };
|
|
8796
8881
|
}
|
|
8797
8882
|
|
|
8798
|
-
var
|
|
8883
|
+
var userManaged = /*#__PURE__*/Object.freeze({
|
|
8799
8884
|
__proto__: null,
|
|
8800
|
-
handleGet: handleGet$
|
|
8801
|
-
handleInsert: handleInsert$
|
|
8802
|
-
handleUpdate: handleUpdate$
|
|
8803
|
-
handleUpsert: handleUpsert$
|
|
8885
|
+
handleGet: handleGet$3,
|
|
8886
|
+
handleInsert: handleInsert$3,
|
|
8887
|
+
handleUpdate: handleUpdate$3,
|
|
8888
|
+
handleUpsert: handleUpsert$3
|
|
8804
8889
|
});
|
|
8805
8890
|
|
|
8806
8891
|
const TRUNCATE_SUFFIX = "...";
|
|
8807
8892
|
const TRUNCATE_SUFFIX_BYTES = calculateUTF8Bytes(TRUNCATE_SUFFIX);
|
|
8808
|
-
async function handleInsert$
|
|
8893
|
+
async function handleInsert$2({ resource, data, mappedData }) {
|
|
8809
8894
|
return handleTruncate({ resource, data, mappedData });
|
|
8810
8895
|
}
|
|
8811
|
-
async function handleUpdate$
|
|
8896
|
+
async function handleUpdate$2({ resource, id, data, mappedData }) {
|
|
8812
8897
|
return handleTruncate({ resource, data, mappedData });
|
|
8813
8898
|
}
|
|
8814
|
-
async function handleUpsert$
|
|
8899
|
+
async function handleUpsert$2({ resource, id, data, mappedData }) {
|
|
8815
8900
|
return handleTruncate({ resource, data, mappedData });
|
|
8816
8901
|
}
|
|
8817
|
-
async function handleGet$
|
|
8902
|
+
async function handleGet$2({ resource, metadata, body }) {
|
|
8818
8903
|
return { metadata, body };
|
|
8819
8904
|
}
|
|
8820
8905
|
function handleTruncate({ resource, data, mappedData }) {
|
|
@@ -8854,25 +8939,25 @@ function handleTruncate({ resource, data, mappedData }) {
|
|
|
8854
8939
|
|
|
8855
8940
|
var dataTruncate = /*#__PURE__*/Object.freeze({
|
|
8856
8941
|
__proto__: null,
|
|
8857
|
-
handleGet: handleGet$
|
|
8858
|
-
handleInsert: handleInsert$
|
|
8859
|
-
handleUpdate: handleUpdate$
|
|
8860
|
-
handleUpsert: handleUpsert$
|
|
8942
|
+
handleGet: handleGet$2,
|
|
8943
|
+
handleInsert: handleInsert$2,
|
|
8944
|
+
handleUpdate: handleUpdate$2,
|
|
8945
|
+
handleUpsert: handleUpsert$2
|
|
8861
8946
|
});
|
|
8862
8947
|
|
|
8863
8948
|
const OVERFLOW_FLAG = "$overflow";
|
|
8864
8949
|
const OVERFLOW_FLAG_VALUE = "true";
|
|
8865
8950
|
const OVERFLOW_FLAG_BYTES = calculateUTF8Bytes(OVERFLOW_FLAG) + calculateUTF8Bytes(OVERFLOW_FLAG_VALUE);
|
|
8866
|
-
async function handleInsert({ resource, data, mappedData }) {
|
|
8951
|
+
async function handleInsert$1({ resource, data, mappedData }) {
|
|
8867
8952
|
return handleOverflow({ resource, data, mappedData });
|
|
8868
8953
|
}
|
|
8869
|
-
async function handleUpdate({ resource, id, data, mappedData }) {
|
|
8954
|
+
async function handleUpdate$1({ resource, id, data, mappedData }) {
|
|
8870
8955
|
return handleOverflow({ resource, data, mappedData });
|
|
8871
8956
|
}
|
|
8872
|
-
async function handleUpsert({ resource, id, data, mappedData }) {
|
|
8957
|
+
async function handleUpsert$1({ resource, id, data, mappedData }) {
|
|
8873
8958
|
return handleOverflow({ resource, data, mappedData });
|
|
8874
8959
|
}
|
|
8875
|
-
async function handleGet({ resource, metadata, body }) {
|
|
8960
|
+
async function handleGet$1({ resource, metadata, body }) {
|
|
8876
8961
|
if (metadata[OVERFLOW_FLAG] === OVERFLOW_FLAG_VALUE) {
|
|
8877
8962
|
try {
|
|
8878
8963
|
const bodyData = body ? JSON.parse(body) : {};
|
|
@@ -8914,6 +8999,51 @@ function handleOverflow({ resource, data, mappedData }) {
|
|
|
8914
8999
|
}
|
|
8915
9000
|
|
|
8916
9001
|
var bodyOverflow = /*#__PURE__*/Object.freeze({
|
|
9002
|
+
__proto__: null,
|
|
9003
|
+
handleGet: handleGet$1,
|
|
9004
|
+
handleInsert: handleInsert$1,
|
|
9005
|
+
handleUpdate: handleUpdate$1,
|
|
9006
|
+
handleUpsert: handleUpsert$1
|
|
9007
|
+
});
|
|
9008
|
+
|
|
9009
|
+
async function handleInsert({ resource, data, mappedData }) {
|
|
9010
|
+
const bodyContent = JSON.stringify(mappedData);
|
|
9011
|
+
return {
|
|
9012
|
+
mappedData: {},
|
|
9013
|
+
body: bodyContent
|
|
9014
|
+
};
|
|
9015
|
+
}
|
|
9016
|
+
async function handleUpdate({ resource, id, data, mappedData }) {
|
|
9017
|
+
const bodyContent = JSON.stringify(mappedData);
|
|
9018
|
+
return {
|
|
9019
|
+
mappedData: {},
|
|
9020
|
+
body: bodyContent
|
|
9021
|
+
};
|
|
9022
|
+
}
|
|
9023
|
+
async function handleUpsert({ resource, id, data, mappedData }) {
|
|
9024
|
+
const bodyContent = JSON.stringify(mappedData);
|
|
9025
|
+
return {
|
|
9026
|
+
mappedData: {},
|
|
9027
|
+
body: bodyContent
|
|
9028
|
+
};
|
|
9029
|
+
}
|
|
9030
|
+
async function handleGet({ resource, metadata, body }) {
|
|
9031
|
+
try {
|
|
9032
|
+
const bodyData = body ? JSON.parse(body) : {};
|
|
9033
|
+
return {
|
|
9034
|
+
metadata: bodyData,
|
|
9035
|
+
body: ""
|
|
9036
|
+
};
|
|
9037
|
+
} catch (error) {
|
|
9038
|
+
console.warn(`Failed to parse body-only content:`, error.message);
|
|
9039
|
+
return {
|
|
9040
|
+
metadata,
|
|
9041
|
+
body: ""
|
|
9042
|
+
};
|
|
9043
|
+
}
|
|
9044
|
+
}
|
|
9045
|
+
|
|
9046
|
+
var bodyOnly = /*#__PURE__*/Object.freeze({
|
|
8917
9047
|
__proto__: null,
|
|
8918
9048
|
handleGet: handleGet,
|
|
8919
9049
|
handleInsert: handleInsert,
|
|
@@ -8922,10 +9052,11 @@ var bodyOverflow = /*#__PURE__*/Object.freeze({
|
|
|
8922
9052
|
});
|
|
8923
9053
|
|
|
8924
9054
|
const behaviors = {
|
|
8925
|
-
"user-
|
|
9055
|
+
"user-managed": userManaged,
|
|
8926
9056
|
"enforce-limits": enforceLimits,
|
|
8927
9057
|
"data-truncate": dataTruncate,
|
|
8928
|
-
"body-overflow": bodyOverflow
|
|
9058
|
+
"body-overflow": bodyOverflow,
|
|
9059
|
+
"body-only": bodyOnly
|
|
8929
9060
|
};
|
|
8930
9061
|
function getBehavior(behaviorName) {
|
|
8931
9062
|
const behavior = behaviors[behaviorName];
|
|
@@ -8935,8 +9066,11 @@ function getBehavior(behaviorName) {
|
|
|
8935
9066
|
return behavior;
|
|
8936
9067
|
}
|
|
8937
9068
|
const AVAILABLE_BEHAVIORS = Object.keys(behaviors);
|
|
8938
|
-
const DEFAULT_BEHAVIOR = "user-
|
|
9069
|
+
const DEFAULT_BEHAVIOR = "user-managed";
|
|
8939
9070
|
|
|
9071
|
+
function createIdGeneratorWithSize(size) {
|
|
9072
|
+
return customAlphabet(urlAlphabet, size);
|
|
9073
|
+
}
|
|
8940
9074
|
class Resource extends EventEmitter {
|
|
8941
9075
|
/**
|
|
8942
9076
|
* Create a new Resource instance
|
|
@@ -8945,7 +9079,7 @@ class Resource extends EventEmitter {
|
|
|
8945
9079
|
* @param {Object} config.client - S3 client instance
|
|
8946
9080
|
* @param {string} [config.version='v0'] - Resource version
|
|
8947
9081
|
* @param {Object} [config.attributes={}] - Resource attributes schema
|
|
8948
|
-
* @param {string} [config.behavior='user-
|
|
9082
|
+
* @param {string} [config.behavior='user-managed'] - Resource behavior strategy
|
|
8949
9083
|
* @param {string} [config.passphrase='secret'] - Encryption passphrase
|
|
8950
9084
|
* @param {number} [config.parallelism=10] - Parallelism for bulk operations
|
|
8951
9085
|
* @param {Array} [config.observers=[]] - Observer instances
|
|
@@ -8957,6 +9091,9 @@ class Resource extends EventEmitter {
|
|
|
8957
9091
|
* @param {boolean} [config.allNestedObjectsOptional=false] - Make nested objects optional
|
|
8958
9092
|
* @param {Object} [config.hooks={}] - Custom hooks
|
|
8959
9093
|
* @param {Object} [config.options={}] - Additional options
|
|
9094
|
+
* @param {Function} [config.idGenerator] - Custom ID generator function
|
|
9095
|
+
* @param {number} [config.idSize=22] - Size for auto-generated IDs
|
|
9096
|
+
* @param {boolean} [config.versioningEnabled=false] - Enable versioning for this resource
|
|
8960
9097
|
* @example
|
|
8961
9098
|
* const users = new Resource({
|
|
8962
9099
|
* name: 'users',
|
|
@@ -8966,7 +9103,7 @@ class Resource extends EventEmitter {
|
|
|
8966
9103
|
* email: 'string|required',
|
|
8967
9104
|
* password: 'secret|required'
|
|
8968
9105
|
* },
|
|
8969
|
-
* behavior: 'user-
|
|
9106
|
+
* behavior: 'user-managed',
|
|
8970
9107
|
* passphrase: 'my-secret-key',
|
|
8971
9108
|
* timestamps: true,
|
|
8972
9109
|
* partitions: {
|
|
@@ -8981,6 +9118,30 @@ class Resource extends EventEmitter {
|
|
|
8981
9118
|
* }]
|
|
8982
9119
|
* }
|
|
8983
9120
|
* });
|
|
9121
|
+
*
|
|
9122
|
+
* // With custom ID size
|
|
9123
|
+
* const shortIdUsers = new Resource({
|
|
9124
|
+
* name: 'users',
|
|
9125
|
+
* client: s3Client,
|
|
9126
|
+
* attributes: { name: 'string|required' },
|
|
9127
|
+
* idSize: 8 // Generate 8-character IDs
|
|
9128
|
+
* });
|
|
9129
|
+
*
|
|
9130
|
+
* // With custom ID generator function
|
|
9131
|
+
* const customIdUsers = new Resource({
|
|
9132
|
+
* name: 'users',
|
|
9133
|
+
* client: s3Client,
|
|
9134
|
+
* attributes: { name: 'string|required' },
|
|
9135
|
+
* idGenerator: () => `user_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`
|
|
9136
|
+
* });
|
|
9137
|
+
*
|
|
9138
|
+
* // With custom ID generator using size parameter
|
|
9139
|
+
* const longIdUsers = new Resource({
|
|
9140
|
+
* name: 'users',
|
|
9141
|
+
* client: s3Client,
|
|
9142
|
+
* attributes: { name: 'string|required' },
|
|
9143
|
+
* idGenerator: 32 // Generate 32-character IDs (same as idSize: 32)
|
|
9144
|
+
* });
|
|
8984
9145
|
*/
|
|
8985
9146
|
constructor(config) {
|
|
8986
9147
|
super();
|
|
@@ -9004,7 +9165,10 @@ ${validation.errors.join("\n")}`);
|
|
|
9004
9165
|
partitions = {},
|
|
9005
9166
|
paranoid = true,
|
|
9006
9167
|
allNestedObjectsOptional = true,
|
|
9007
|
-
hooks = {}
|
|
9168
|
+
hooks = {},
|
|
9169
|
+
idGenerator: customIdGenerator,
|
|
9170
|
+
idSize = 22,
|
|
9171
|
+
versioningEnabled = false
|
|
9008
9172
|
} = config;
|
|
9009
9173
|
this.name = name;
|
|
9010
9174
|
this.client = client;
|
|
@@ -9013,6 +9177,8 @@ ${validation.errors.join("\n")}`);
|
|
|
9013
9177
|
this.observers = observers;
|
|
9014
9178
|
this.parallelism = parallelism;
|
|
9015
9179
|
this.passphrase = passphrase ?? "secret";
|
|
9180
|
+
this.versioningEnabled = versioningEnabled;
|
|
9181
|
+
this.idGenerator = this.configureIdGenerator(customIdGenerator, idSize);
|
|
9016
9182
|
this.config = {
|
|
9017
9183
|
cache,
|
|
9018
9184
|
hooks,
|
|
@@ -9044,6 +9210,25 @@ ${validation.errors.join("\n")}`);
|
|
|
9044
9210
|
}
|
|
9045
9211
|
}
|
|
9046
9212
|
}
|
|
9213
|
+
/**
|
|
9214
|
+
* Configure ID generator based on provided options
|
|
9215
|
+
* @param {Function|number} customIdGenerator - Custom ID generator function or size
|
|
9216
|
+
* @param {number} idSize - Size for auto-generated IDs
|
|
9217
|
+
* @returns {Function} Configured ID generator function
|
|
9218
|
+
* @private
|
|
9219
|
+
*/
|
|
9220
|
+
configureIdGenerator(customIdGenerator, idSize) {
|
|
9221
|
+
if (typeof customIdGenerator === "function") {
|
|
9222
|
+
return customIdGenerator;
|
|
9223
|
+
}
|
|
9224
|
+
if (typeof customIdGenerator === "number" && customIdGenerator > 0) {
|
|
9225
|
+
return createIdGeneratorWithSize(customIdGenerator);
|
|
9226
|
+
}
|
|
9227
|
+
if (typeof idSize === "number" && idSize > 0) {
|
|
9228
|
+
return createIdGeneratorWithSize(idSize);
|
|
9229
|
+
}
|
|
9230
|
+
return idGenerator;
|
|
9231
|
+
}
|
|
9047
9232
|
/**
|
|
9048
9233
|
* Get resource options (for backward compatibility with tests)
|
|
9049
9234
|
*/
|
|
@@ -9100,6 +9285,15 @@ ${validation.errors.join("\n")}`);
|
|
|
9100
9285
|
}
|
|
9101
9286
|
}
|
|
9102
9287
|
this.setupPartitionHooks();
|
|
9288
|
+
if (this.versioningEnabled) {
|
|
9289
|
+
if (!this.config.partitions.byVersion) {
|
|
9290
|
+
this.config.partitions.byVersion = {
|
|
9291
|
+
fields: {
|
|
9292
|
+
_v: "string"
|
|
9293
|
+
}
|
|
9294
|
+
};
|
|
9295
|
+
}
|
|
9296
|
+
}
|
|
9103
9297
|
this.schema = new Schema({
|
|
9104
9298
|
name: this.name,
|
|
9105
9299
|
attributes: this.attributes,
|
|
@@ -9219,6 +9413,9 @@ ${validation.errors.join("\n")}`);
|
|
|
9219
9413
|
* @returns {boolean} True if field exists
|
|
9220
9414
|
*/
|
|
9221
9415
|
fieldExistsInAttributes(fieldName) {
|
|
9416
|
+
if (fieldName.startsWith("_")) {
|
|
9417
|
+
return true;
|
|
9418
|
+
}
|
|
9222
9419
|
if (!fieldName.includes(".")) {
|
|
9223
9420
|
return Object.keys(this.attributes || {}).includes(fieldName);
|
|
9224
9421
|
}
|
|
@@ -9272,12 +9469,12 @@ ${validation.errors.join("\n")}`);
|
|
|
9272
9469
|
return transformedValue;
|
|
9273
9470
|
}
|
|
9274
9471
|
/**
|
|
9275
|
-
* Get the main resource key (
|
|
9472
|
+
* Get the main resource key (new format without version in path)
|
|
9276
9473
|
* @param {string} id - Resource ID
|
|
9277
9474
|
* @returns {string} The main S3 key path
|
|
9278
9475
|
*/
|
|
9279
9476
|
getResourceKey(id) {
|
|
9280
|
-
return join(`resource=${this.name}`, `
|
|
9477
|
+
return join(`resource=${this.name}`, `data`, `id=${id}`);
|
|
9281
9478
|
}
|
|
9282
9479
|
/**
|
|
9283
9480
|
* Generate partition key for a resource in a specific partition
|
|
@@ -9346,12 +9543,23 @@ ${validation.errors.join("\n")}`);
|
|
|
9346
9543
|
}
|
|
9347
9544
|
return currentLevel;
|
|
9348
9545
|
}
|
|
9546
|
+
/**
|
|
9547
|
+
* Calculate estimated content length for body data
|
|
9548
|
+
* @param {string|Buffer} body - Body content
|
|
9549
|
+
* @returns {number} Estimated content length in bytes
|
|
9550
|
+
*/
|
|
9551
|
+
calculateContentLength(body) {
|
|
9552
|
+
if (!body) return 0;
|
|
9553
|
+
if (Buffer.isBuffer(body)) return body.length;
|
|
9554
|
+
if (typeof body === "string") return Buffer.byteLength(body, "utf8");
|
|
9555
|
+
if (typeof body === "object") return Buffer.byteLength(JSON.stringify(body), "utf8");
|
|
9556
|
+
return Buffer.byteLength(String(body), "utf8");
|
|
9557
|
+
}
|
|
9349
9558
|
/**
|
|
9350
9559
|
* Insert a new resource object
|
|
9351
|
-
* @param {Object}
|
|
9352
|
-
* @param {string} [
|
|
9353
|
-
* @
|
|
9354
|
-
* @returns {Promise<Object>} The inserted resource object with all attributes and generated ID
|
|
9560
|
+
* @param {Object} attributes - Resource attributes
|
|
9561
|
+
* @param {string} [attributes.id] - Custom ID (optional, auto-generated if not provided)
|
|
9562
|
+
* @returns {Promise<Object>} The created resource object with all attributes
|
|
9355
9563
|
* @example
|
|
9356
9564
|
* // Insert with auto-generated ID
|
|
9357
9565
|
* const user = await resource.insert({
|
|
@@ -9359,18 +9567,13 @@ ${validation.errors.join("\n")}`);
|
|
|
9359
9567
|
* email: 'john@example.com',
|
|
9360
9568
|
* age: 30
|
|
9361
9569
|
* });
|
|
9570
|
+
* console.log(user.id); // Auto-generated ID
|
|
9362
9571
|
*
|
|
9363
9572
|
* // Insert with custom ID
|
|
9364
9573
|
* const user = await resource.insert({
|
|
9365
|
-
* id: '
|
|
9366
|
-
* name: 'Jane Smith',
|
|
9367
|
-
* email: 'jane@example.com'
|
|
9368
|
-
* });
|
|
9369
|
-
*
|
|
9370
|
-
* // Insert with auto-generated password for secret field
|
|
9371
|
-
* const user = await resource.insert({
|
|
9574
|
+
* id: 'user-123',
|
|
9372
9575
|
* name: 'John Doe',
|
|
9373
|
-
* email: 'john@example.com'
|
|
9576
|
+
* email: 'john@example.com'
|
|
9374
9577
|
* });
|
|
9375
9578
|
*/
|
|
9376
9579
|
async insert({ id, ...attributes }) {
|
|
@@ -9378,7 +9581,8 @@ ${validation.errors.join("\n")}`);
|
|
|
9378
9581
|
attributes.createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9379
9582
|
attributes.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9380
9583
|
}
|
|
9381
|
-
const
|
|
9584
|
+
const completeData = { id, ...attributes };
|
|
9585
|
+
const preProcessedData = await this.executeHooks("preInsert", completeData);
|
|
9382
9586
|
const {
|
|
9383
9587
|
errors,
|
|
9384
9588
|
isValid,
|
|
@@ -9392,21 +9596,34 @@ ${validation.errors.join("\n")}`);
|
|
|
9392
9596
|
validation: errors
|
|
9393
9597
|
});
|
|
9394
9598
|
}
|
|
9395
|
-
|
|
9396
|
-
const
|
|
9599
|
+
const { id: validatedId, ...validatedAttributes } = validated;
|
|
9600
|
+
const finalId = validatedId || id || this.idGenerator();
|
|
9601
|
+
const mappedData = await this.schema.mapper(validatedAttributes);
|
|
9602
|
+
mappedData._v = String(this.version);
|
|
9397
9603
|
const behaviorImpl = getBehavior(this.behavior);
|
|
9398
9604
|
const { mappedData: processedMetadata, body } = await behaviorImpl.handleInsert({
|
|
9399
9605
|
resource: this,
|
|
9400
|
-
data:
|
|
9606
|
+
data: validatedAttributes,
|
|
9401
9607
|
mappedData
|
|
9402
9608
|
});
|
|
9403
|
-
const
|
|
9609
|
+
const finalMetadata = processedMetadata;
|
|
9610
|
+
const key = this.getResourceKey(finalId);
|
|
9611
|
+
let contentType = void 0;
|
|
9612
|
+
if (body && body !== "") {
|
|
9613
|
+
try {
|
|
9614
|
+
JSON.parse(body);
|
|
9615
|
+
contentType = "application/json";
|
|
9616
|
+
} catch {
|
|
9617
|
+
}
|
|
9618
|
+
}
|
|
9404
9619
|
await this.client.putObject({
|
|
9405
|
-
metadata:
|
|
9620
|
+
metadata: finalMetadata,
|
|
9406
9621
|
key,
|
|
9407
|
-
body
|
|
9622
|
+
body,
|
|
9623
|
+
contentType,
|
|
9624
|
+
contentLength: this.calculateContentLength(body)
|
|
9408
9625
|
});
|
|
9409
|
-
const final = merge({ id },
|
|
9626
|
+
const final = merge({ id: finalId }, validatedAttributes);
|
|
9410
9627
|
await this.executeHooks("afterInsert", final);
|
|
9411
9628
|
this.emit("insert", final);
|
|
9412
9629
|
return final;
|
|
@@ -9425,7 +9642,8 @@ ${validation.errors.join("\n")}`);
|
|
|
9425
9642
|
const key = this.getResourceKey(id);
|
|
9426
9643
|
try {
|
|
9427
9644
|
const request = await this.client.headObject(key);
|
|
9428
|
-
const
|
|
9645
|
+
const objectVersionRaw = request.Metadata?._v || this.version;
|
|
9646
|
+
const objectVersion = typeof objectVersionRaw === "string" && objectVersionRaw.startsWith("v") ? objectVersionRaw.slice(1) : objectVersionRaw;
|
|
9429
9647
|
const schema = await this.getSchemaForVersion(objectVersion);
|
|
9430
9648
|
let metadata = await schema.unmapper(request.Metadata);
|
|
9431
9649
|
const behaviorImpl = getBehavior(this.behavior);
|
|
@@ -9450,9 +9668,13 @@ ${validation.errors.join("\n")}`);
|
|
|
9450
9668
|
data._lastModified = request.LastModified;
|
|
9451
9669
|
data._hasContent = request.ContentLength > 0;
|
|
9452
9670
|
data._mimeType = request.ContentType || null;
|
|
9671
|
+
data._v = objectVersion;
|
|
9453
9672
|
if (request.VersionId) data._versionId = request.VersionId;
|
|
9454
9673
|
if (request.Expiration) data._expiresAt = request.Expiration;
|
|
9455
9674
|
data._definitionHash = this.getDefinitionHash();
|
|
9675
|
+
if (objectVersion !== this.version) {
|
|
9676
|
+
data = await this.applyVersionMapping(data, objectVersion, this.version);
|
|
9677
|
+
}
|
|
9456
9678
|
this.emit("get", data);
|
|
9457
9679
|
return data;
|
|
9458
9680
|
} catch (error) {
|
|
@@ -9558,28 +9780,30 @@ ${validation.errors.join("\n")}`);
|
|
|
9558
9780
|
attributes.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9559
9781
|
}
|
|
9560
9782
|
const preProcessedData = await this.executeHooks("preUpdate", attributes);
|
|
9561
|
-
const
|
|
9562
|
-
|
|
9563
|
-
const { isValid, errors, data: validated } = await this.validate(attrs);
|
|
9783
|
+
const completeData = { ...originalData, ...preProcessedData, id };
|
|
9784
|
+
const { isValid, errors, data: validated } = await this.validate(completeData);
|
|
9564
9785
|
if (!isValid) {
|
|
9565
9786
|
throw new InvalidResourceItem({
|
|
9566
|
-
bucket: this.client.bucket,
|
|
9787
|
+
bucket: this.client.config.bucket,
|
|
9567
9788
|
resourceName: this.name,
|
|
9568
9789
|
attributes: preProcessedData,
|
|
9569
9790
|
validation: errors
|
|
9570
9791
|
});
|
|
9571
9792
|
}
|
|
9793
|
+
const { id: validatedId, ...validatedAttributes } = validated;
|
|
9572
9794
|
const oldData = { ...originalData, id };
|
|
9573
|
-
const newData = { ...
|
|
9795
|
+
const newData = { ...validatedAttributes, id };
|
|
9574
9796
|
await this.handlePartitionReferenceUpdates(oldData, newData);
|
|
9575
|
-
const mappedData = await this.schema.mapper(
|
|
9797
|
+
const mappedData = await this.schema.mapper(validatedAttributes);
|
|
9798
|
+
mappedData._v = String(this.version);
|
|
9576
9799
|
const behaviorImpl = getBehavior(this.behavior);
|
|
9577
9800
|
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
9578
9801
|
resource: this,
|
|
9579
9802
|
id,
|
|
9580
|
-
data:
|
|
9803
|
+
data: validatedAttributes,
|
|
9581
9804
|
mappedData
|
|
9582
9805
|
});
|
|
9806
|
+
const finalMetadata = processedMetadata;
|
|
9583
9807
|
const key = this.getResourceKey(id);
|
|
9584
9808
|
let existingContentType = void 0;
|
|
9585
9809
|
let finalBody = body;
|
|
@@ -9599,16 +9823,27 @@ ${validation.errors.join("\n")}`);
|
|
|
9599
9823
|
} catch (error) {
|
|
9600
9824
|
}
|
|
9601
9825
|
}
|
|
9826
|
+
let finalContentType = existingContentType;
|
|
9827
|
+
if (finalBody && finalBody !== "" && !finalContentType) {
|
|
9828
|
+
try {
|
|
9829
|
+
JSON.parse(finalBody);
|
|
9830
|
+
finalContentType = "application/json";
|
|
9831
|
+
} catch {
|
|
9832
|
+
}
|
|
9833
|
+
}
|
|
9834
|
+
if (this.versioningEnabled && originalData._v !== this.version) {
|
|
9835
|
+
await this.createHistoricalVersion(id, originalData);
|
|
9836
|
+
}
|
|
9602
9837
|
await this.client.putObject({
|
|
9603
9838
|
key,
|
|
9604
9839
|
body: finalBody,
|
|
9605
|
-
contentType:
|
|
9606
|
-
metadata:
|
|
9840
|
+
contentType: finalContentType,
|
|
9841
|
+
metadata: finalMetadata
|
|
9607
9842
|
});
|
|
9608
|
-
|
|
9609
|
-
await this.executeHooks("afterUpdate",
|
|
9610
|
-
this.emit("update", preProcessedData,
|
|
9611
|
-
return
|
|
9843
|
+
validatedAttributes.id = id;
|
|
9844
|
+
await this.executeHooks("afterUpdate", validatedAttributes);
|
|
9845
|
+
this.emit("update", preProcessedData, validatedAttributes);
|
|
9846
|
+
return validatedAttributes;
|
|
9612
9847
|
}
|
|
9613
9848
|
/**
|
|
9614
9849
|
* Delete a resource object by ID
|
|
@@ -9697,11 +9932,9 @@ ${validation.errors.join("\n")}`);
|
|
|
9697
9932
|
prefix = `resource=${this.name}/partition=${partition}`;
|
|
9698
9933
|
}
|
|
9699
9934
|
} else {
|
|
9700
|
-
prefix = `resource=${this.name}/
|
|
9935
|
+
prefix = `resource=${this.name}/data`;
|
|
9701
9936
|
}
|
|
9702
|
-
const count = await this.client.count({
|
|
9703
|
-
prefix
|
|
9704
|
-
});
|
|
9937
|
+
const count = await this.client.count({ prefix });
|
|
9705
9938
|
this.emit("count", count);
|
|
9706
9939
|
return count;
|
|
9707
9940
|
}
|
|
@@ -9768,7 +10001,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9768
10001
|
`deleteAll() is a dangerous operation and requires paranoid: false option. Current paranoid setting: ${this.config.paranoid}`
|
|
9769
10002
|
);
|
|
9770
10003
|
}
|
|
9771
|
-
const prefix = `resource=${this.name}/
|
|
10004
|
+
const prefix = `resource=${this.name}/data`;
|
|
9772
10005
|
const deletedCount = await this.client.deleteAll({ prefix });
|
|
9773
10006
|
this.emit("deleteAll", {
|
|
9774
10007
|
version: this.version,
|
|
@@ -9846,7 +10079,7 @@ ${validation.errors.join("\n")}`);
|
|
|
9846
10079
|
prefix = `resource=${this.name}/partition=${partition}`;
|
|
9847
10080
|
}
|
|
9848
10081
|
} else {
|
|
9849
|
-
prefix = `resource=${this.name}/
|
|
10082
|
+
prefix = `resource=${this.name}/data`;
|
|
9850
10083
|
}
|
|
9851
10084
|
const keys = await this.client.getKeysPage({
|
|
9852
10085
|
prefix,
|
|
@@ -9863,200 +10096,221 @@ ${validation.errors.join("\n")}`);
|
|
|
9863
10096
|
return ids;
|
|
9864
10097
|
}
|
|
9865
10098
|
/**
|
|
9866
|
-
* List
|
|
10099
|
+
* List resources with optional partition filtering and pagination
|
|
9867
10100
|
* @param {Object} [params] - List parameters
|
|
9868
10101
|
* @param {string} [params.partition] - Partition name to list from
|
|
9869
10102
|
* @param {Object} [params.partitionValues] - Partition field values to filter by
|
|
9870
|
-
* @param {number} [params.limit] - Maximum number of results
|
|
9871
|
-
* @param {number} [params.offset=0] -
|
|
9872
|
-
* @returns {Promise<Object[]>} Array of resource objects
|
|
10103
|
+
* @param {number} [params.limit] - Maximum number of results
|
|
10104
|
+
* @param {number} [params.offset=0] - Number of results to skip
|
|
10105
|
+
* @returns {Promise<Object[]>} Array of resource objects
|
|
9873
10106
|
* @example
|
|
9874
10107
|
* // List all resources
|
|
9875
10108
|
* const allUsers = await resource.list();
|
|
9876
10109
|
*
|
|
9877
10110
|
* // List with pagination
|
|
9878
|
-
* const
|
|
9879
|
-
* const secondPage = await resource.list({ limit: 10, offset: 10 });
|
|
10111
|
+
* const first10 = await resource.list({ limit: 10, offset: 0 });
|
|
9880
10112
|
*
|
|
9881
10113
|
* // List from specific partition
|
|
9882
|
-
* const
|
|
9883
|
-
* partition: '
|
|
9884
|
-
* partitionValues: { '
|
|
9885
|
-
* });
|
|
9886
|
-
*
|
|
9887
|
-
* // List from partition with pagination
|
|
9888
|
-
* const googleUsersPage = await resource.list({
|
|
9889
|
-
* partition: 'byUtmSource',
|
|
9890
|
-
* partitionValues: { 'utm.source': 'google' },
|
|
9891
|
-
* limit: 5,
|
|
9892
|
-
* offset: 0
|
|
10114
|
+
* const usUsers = await resource.list({
|
|
10115
|
+
* partition: 'byCountry',
|
|
10116
|
+
* partitionValues: { 'profile.country': 'US' }
|
|
9893
10117
|
* });
|
|
9894
10118
|
*/
|
|
9895
10119
|
async list({ partition = null, partitionValues = {}, limit, offset = 0 } = {}) {
|
|
9896
10120
|
try {
|
|
9897
10121
|
if (!partition) {
|
|
9898
|
-
|
|
9899
|
-
try {
|
|
9900
|
-
ids2 = await this.listIds({ partition, partitionValues });
|
|
9901
|
-
} catch (listIdsError) {
|
|
9902
|
-
console.warn(`Failed to get list IDs:`, listIdsError.message);
|
|
9903
|
-
return [];
|
|
9904
|
-
}
|
|
9905
|
-
let filteredIds2 = ids2.slice(offset);
|
|
9906
|
-
if (limit) {
|
|
9907
|
-
filteredIds2 = filteredIds2.slice(0, limit);
|
|
9908
|
-
}
|
|
9909
|
-
const { results: results2, errors: errors2 } = await PromisePool.for(filteredIds2).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
9910
|
-
console.warn(`Failed to get resource ${id}:`, error.message);
|
|
9911
|
-
return null;
|
|
9912
|
-
}).process(async (id) => {
|
|
9913
|
-
try {
|
|
9914
|
-
return await this.get(id);
|
|
9915
|
-
} catch (error) {
|
|
9916
|
-
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
9917
|
-
console.warn(`Decryption failed for ${id}, returning basic info`);
|
|
9918
|
-
return {
|
|
9919
|
-
id,
|
|
9920
|
-
_decryptionFailed: true,
|
|
9921
|
-
_error: error.message
|
|
9922
|
-
};
|
|
9923
|
-
}
|
|
9924
|
-
throw error;
|
|
9925
|
-
}
|
|
9926
|
-
});
|
|
9927
|
-
const validResults2 = results2.filter((item) => item !== null);
|
|
9928
|
-
this.emit("list", { partition, partitionValues, count: validResults2.length, errors: errors2.length });
|
|
9929
|
-
return validResults2;
|
|
9930
|
-
}
|
|
9931
|
-
if (!this.config.partitions || !this.config.partitions[partition]) {
|
|
9932
|
-
console.warn(`Partition '${partition}' not found in resource '${this.name}'`);
|
|
9933
|
-
this.emit("list", { partition, partitionValues, count: 0, errors: 0 });
|
|
9934
|
-
return [];
|
|
9935
|
-
}
|
|
9936
|
-
const partitionDef = this.config.partitions[partition];
|
|
9937
|
-
const partitionSegments = [];
|
|
9938
|
-
const sortedFields = Object.entries(partitionDef.fields).sort(([a], [b]) => a.localeCompare(b));
|
|
9939
|
-
for (const [fieldName, rule] of sortedFields) {
|
|
9940
|
-
const value = partitionValues[fieldName];
|
|
9941
|
-
if (value !== void 0 && value !== null) {
|
|
9942
|
-
const transformedValue = this.applyPartitionRule(value, rule);
|
|
9943
|
-
partitionSegments.push(`${fieldName}=${transformedValue}`);
|
|
9944
|
-
}
|
|
9945
|
-
}
|
|
9946
|
-
let prefix;
|
|
9947
|
-
if (partitionSegments.length > 0) {
|
|
9948
|
-
prefix = `resource=${this.name}/partition=${partition}/${partitionSegments.join("/")}`;
|
|
9949
|
-
} else {
|
|
9950
|
-
prefix = `resource=${this.name}/partition=${partition}`;
|
|
9951
|
-
}
|
|
9952
|
-
let keys = [];
|
|
9953
|
-
try {
|
|
9954
|
-
keys = await this.client.getAllKeys({ prefix });
|
|
9955
|
-
} catch (getKeysError) {
|
|
9956
|
-
console.warn(`Failed to get partition keys:`, getKeysError.message);
|
|
9957
|
-
return [];
|
|
10122
|
+
return await this.listMain({ limit, offset });
|
|
9958
10123
|
}
|
|
9959
|
-
|
|
9960
|
-
const parts = key.split("/");
|
|
9961
|
-
const idPart = parts.find((part) => part.startsWith("id="));
|
|
9962
|
-
return idPart ? idPart.replace("id=", "") : null;
|
|
9963
|
-
}).filter(Boolean);
|
|
9964
|
-
let filteredIds = ids.slice(offset);
|
|
9965
|
-
if (limit) {
|
|
9966
|
-
filteredIds = filteredIds.slice(0, limit);
|
|
9967
|
-
}
|
|
9968
|
-
const { results, errors } = await PromisePool.for(filteredIds).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
9969
|
-
console.warn(`Failed to get partition resource ${id}:`, error.message);
|
|
9970
|
-
return null;
|
|
9971
|
-
}).process(async (id) => {
|
|
9972
|
-
try {
|
|
9973
|
-
const keyForId = keys.find((key) => key.includes(`id=${id}`));
|
|
9974
|
-
if (!keyForId) {
|
|
9975
|
-
throw new Error(`Partition key not found for ID ${id}`);
|
|
9976
|
-
}
|
|
9977
|
-
const keyParts = keyForId.split("/");
|
|
9978
|
-
const actualPartitionValues = {};
|
|
9979
|
-
for (const [fieldName, rule] of sortedFields) {
|
|
9980
|
-
const fieldPart = keyParts.find((part) => part.startsWith(`${fieldName}=`));
|
|
9981
|
-
if (fieldPart) {
|
|
9982
|
-
const value = fieldPart.replace(`${fieldName}=`, "");
|
|
9983
|
-
actualPartitionValues[fieldName] = value;
|
|
9984
|
-
}
|
|
9985
|
-
}
|
|
9986
|
-
return await this.getFromPartition({ id, partitionName: partition, partitionValues: actualPartitionValues });
|
|
9987
|
-
} catch (error) {
|
|
9988
|
-
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
9989
|
-
console.warn(`Decryption failed for partition resource ${id}, returning basic info`);
|
|
9990
|
-
return {
|
|
9991
|
-
id,
|
|
9992
|
-
_partition: partition,
|
|
9993
|
-
_partitionValues: partitionValues,
|
|
9994
|
-
_decryptionFailed: true,
|
|
9995
|
-
_error: error.message
|
|
9996
|
-
};
|
|
9997
|
-
}
|
|
9998
|
-
throw error;
|
|
9999
|
-
}
|
|
10000
|
-
});
|
|
10001
|
-
const validResults = results.filter((item) => item !== null);
|
|
10002
|
-
this.emit("list", { partition, partitionValues, count: validResults.length, errors: errors.length });
|
|
10003
|
-
return validResults;
|
|
10124
|
+
return await this.listPartition({ partition, partitionValues, limit, offset });
|
|
10004
10125
|
} catch (error) {
|
|
10005
|
-
|
|
10006
|
-
console.warn(`Partition error in list method:`, error.message);
|
|
10007
|
-
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10008
|
-
return [];
|
|
10009
|
-
}
|
|
10010
|
-
console.error(`Critical error in list method:`, error.message);
|
|
10011
|
-
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10012
|
-
return [];
|
|
10126
|
+
return this.handleListError(error, { partition, partitionValues });
|
|
10013
10127
|
}
|
|
10014
10128
|
}
|
|
10015
10129
|
/**
|
|
10016
|
-
*
|
|
10017
|
-
* @param {string[]} ids - Array of resource IDs
|
|
10018
|
-
* @returns {Promise<Object[]>} Array of resource objects
|
|
10019
|
-
* @example
|
|
10020
|
-
* const users = await resource.getMany(['user-1', 'user-2', 'user-3']);
|
|
10021
|
-
* users.forEach(user => console.log(user.name));
|
|
10130
|
+
* List resources from main resource (no partition)
|
|
10022
10131
|
*/
|
|
10023
|
-
async
|
|
10024
|
-
const
|
|
10025
|
-
|
|
10026
|
-
|
|
10027
|
-
id,
|
|
10028
|
-
_error: error.message,
|
|
10029
|
-
_decryptionFailed: error.message.includes("Cipher job failed") || error.message.includes("OperationError")
|
|
10030
|
-
};
|
|
10031
|
-
}).process(async (id) => {
|
|
10032
|
-
this.emit("id", id);
|
|
10033
|
-
try {
|
|
10034
|
-
const data = await this.get(id);
|
|
10035
|
-
this.emit("data", data);
|
|
10036
|
-
return data;
|
|
10037
|
-
} catch (error) {
|
|
10038
|
-
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
10039
|
-
console.warn(`Decryption failed for ${id}, returning basic info`);
|
|
10040
|
-
return {
|
|
10041
|
-
id,
|
|
10042
|
-
_decryptionFailed: true,
|
|
10043
|
-
_error: error.message
|
|
10044
|
-
};
|
|
10045
|
-
}
|
|
10046
|
-
throw error;
|
|
10047
|
-
}
|
|
10048
|
-
});
|
|
10049
|
-
this.emit("getMany", ids.length);
|
|
10132
|
+
async listMain({ limit, offset = 0 }) {
|
|
10133
|
+
const ids = await this.listIds({ limit, offset });
|
|
10134
|
+
const results = await this.processListResults(ids, "main");
|
|
10135
|
+
this.emit("list", { count: results.length, errors: 0 });
|
|
10050
10136
|
return results;
|
|
10051
10137
|
}
|
|
10052
10138
|
/**
|
|
10053
|
-
*
|
|
10054
|
-
* @returns {Promise<Object[]>} Array of all resource objects
|
|
10055
|
-
* @example
|
|
10056
|
-
* const allUsers = await resource.getAll();
|
|
10057
|
-
* console.log(`Total users: ${allUsers.length}`);
|
|
10139
|
+
* List resources from specific partition
|
|
10058
10140
|
*/
|
|
10059
|
-
async
|
|
10141
|
+
async listPartition({ partition, partitionValues, limit, offset = 0 }) {
|
|
10142
|
+
if (!this.config.partitions?.[partition]) {
|
|
10143
|
+
console.warn(`Partition '${partition}' not found in resource '${this.name}'`);
|
|
10144
|
+
this.emit("list", { partition, partitionValues, count: 0, errors: 0 });
|
|
10145
|
+
return [];
|
|
10146
|
+
}
|
|
10147
|
+
const partitionDef = this.config.partitions[partition];
|
|
10148
|
+
const prefix = this.buildPartitionPrefix(partition, partitionDef, partitionValues);
|
|
10149
|
+
const keys = await this.client.getAllKeys({ prefix });
|
|
10150
|
+
const ids = this.extractIdsFromKeys(keys).slice(offset);
|
|
10151
|
+
const filteredIds = limit ? ids.slice(0, limit) : ids;
|
|
10152
|
+
const results = await this.processPartitionResults(filteredIds, partition, partitionDef, keys);
|
|
10153
|
+
this.emit("list", { partition, partitionValues, count: results.length, errors: 0 });
|
|
10154
|
+
return results;
|
|
10155
|
+
}
|
|
10156
|
+
/**
|
|
10157
|
+
* Build partition prefix from partition definition and values
|
|
10158
|
+
*/
|
|
10159
|
+
buildPartitionPrefix(partition, partitionDef, partitionValues) {
|
|
10160
|
+
const partitionSegments = [];
|
|
10161
|
+
const sortedFields = Object.entries(partitionDef.fields).sort(([a], [b]) => a.localeCompare(b));
|
|
10162
|
+
for (const [fieldName, rule] of sortedFields) {
|
|
10163
|
+
const value = partitionValues[fieldName];
|
|
10164
|
+
if (value !== void 0 && value !== null) {
|
|
10165
|
+
const transformedValue = this.applyPartitionRule(value, rule);
|
|
10166
|
+
partitionSegments.push(`${fieldName}=${transformedValue}`);
|
|
10167
|
+
}
|
|
10168
|
+
}
|
|
10169
|
+
if (partitionSegments.length > 0) {
|
|
10170
|
+
return `resource=${this.name}/partition=${partition}/${partitionSegments.join("/")}`;
|
|
10171
|
+
}
|
|
10172
|
+
return `resource=${this.name}/partition=${partition}`;
|
|
10173
|
+
}
|
|
10174
|
+
/**
|
|
10175
|
+
* Extract IDs from S3 keys
|
|
10176
|
+
*/
|
|
10177
|
+
extractIdsFromKeys(keys) {
|
|
10178
|
+
return keys.map((key) => {
|
|
10179
|
+
const parts = key.split("/");
|
|
10180
|
+
const idPart = parts.find((part) => part.startsWith("id="));
|
|
10181
|
+
return idPart ? idPart.replace("id=", "") : null;
|
|
10182
|
+
}).filter(Boolean);
|
|
10183
|
+
}
|
|
10184
|
+
/**
|
|
10185
|
+
* Process list results with error handling
|
|
10186
|
+
*/
|
|
10187
|
+
async processListResults(ids, context = "main") {
|
|
10188
|
+
const { results, errors } = await PromisePool.for(ids).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
10189
|
+
console.warn(`Failed to get ${context} resource ${id}:`, error.message);
|
|
10190
|
+
return null;
|
|
10191
|
+
}).process(async (id) => {
|
|
10192
|
+
try {
|
|
10193
|
+
return await this.get(id);
|
|
10194
|
+
} catch (error) {
|
|
10195
|
+
return this.handleResourceError(error, id, context);
|
|
10196
|
+
}
|
|
10197
|
+
});
|
|
10198
|
+
return results.filter((item) => item !== null);
|
|
10199
|
+
}
|
|
10200
|
+
/**
|
|
10201
|
+
* Process partition results with error handling
|
|
10202
|
+
*/
|
|
10203
|
+
async processPartitionResults(ids, partition, partitionDef, keys) {
|
|
10204
|
+
const sortedFields = Object.entries(partitionDef.fields).sort(([a], [b]) => a.localeCompare(b));
|
|
10205
|
+
const { results, errors } = await PromisePool.for(ids).withConcurrency(this.parallelism).handleError(async (error, id) => {
|
|
10206
|
+
console.warn(`Failed to get partition resource ${id}:`, error.message);
|
|
10207
|
+
return null;
|
|
10208
|
+
}).process(async (id) => {
|
|
10209
|
+
try {
|
|
10210
|
+
const actualPartitionValues = this.extractPartitionValuesFromKey(id, keys, sortedFields);
|
|
10211
|
+
return await this.getFromPartition({
|
|
10212
|
+
id,
|
|
10213
|
+
partitionName: partition,
|
|
10214
|
+
partitionValues: actualPartitionValues
|
|
10215
|
+
});
|
|
10216
|
+
} catch (error) {
|
|
10217
|
+
return this.handleResourceError(error, id, "partition");
|
|
10218
|
+
}
|
|
10219
|
+
});
|
|
10220
|
+
return results.filter((item) => item !== null);
|
|
10221
|
+
}
|
|
10222
|
+
/**
|
|
10223
|
+
* Extract partition values from S3 key for specific ID
|
|
10224
|
+
*/
|
|
10225
|
+
extractPartitionValuesFromKey(id, keys, sortedFields) {
|
|
10226
|
+
const keyForId = keys.find((key) => key.includes(`id=${id}`));
|
|
10227
|
+
if (!keyForId) {
|
|
10228
|
+
throw new Error(`Partition key not found for ID ${id}`);
|
|
10229
|
+
}
|
|
10230
|
+
const keyParts = keyForId.split("/");
|
|
10231
|
+
const actualPartitionValues = {};
|
|
10232
|
+
for (const [fieldName] of sortedFields) {
|
|
10233
|
+
const fieldPart = keyParts.find((part) => part.startsWith(`${fieldName}=`));
|
|
10234
|
+
if (fieldPart) {
|
|
10235
|
+
const value = fieldPart.replace(`${fieldName}=`, "");
|
|
10236
|
+
actualPartitionValues[fieldName] = value;
|
|
10237
|
+
}
|
|
10238
|
+
}
|
|
10239
|
+
return actualPartitionValues;
|
|
10240
|
+
}
|
|
10241
|
+
/**
|
|
10242
|
+
* Handle resource-specific errors
|
|
10243
|
+
*/
|
|
10244
|
+
handleResourceError(error, id, context) {
|
|
10245
|
+
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
10246
|
+
console.warn(`Decryption failed for ${context} resource ${id}, returning basic info`);
|
|
10247
|
+
return {
|
|
10248
|
+
id,
|
|
10249
|
+
_decryptionFailed: true,
|
|
10250
|
+
_error: error.message,
|
|
10251
|
+
...context === "partition" && { _partition: context }
|
|
10252
|
+
};
|
|
10253
|
+
}
|
|
10254
|
+
throw error;
|
|
10255
|
+
}
|
|
10256
|
+
/**
|
|
10257
|
+
* Handle list method errors
|
|
10258
|
+
*/
|
|
10259
|
+
handleListError(error, { partition, partitionValues }) {
|
|
10260
|
+
if (error.message.includes("Partition '") && error.message.includes("' not found")) {
|
|
10261
|
+
console.warn(`Partition error in list method:`, error.message);
|
|
10262
|
+
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10263
|
+
return [];
|
|
10264
|
+
}
|
|
10265
|
+
console.error(`Critical error in list method:`, error.message);
|
|
10266
|
+
this.emit("list", { partition, partitionValues, count: 0, errors: 1 });
|
|
10267
|
+
return [];
|
|
10268
|
+
}
|
|
10269
|
+
/**
|
|
10270
|
+
* Get multiple resources by their IDs
|
|
10271
|
+
* @param {string[]} ids - Array of resource IDs
|
|
10272
|
+
* @returns {Promise<Object[]>} Array of resource objects
|
|
10273
|
+
* @example
|
|
10274
|
+
* const users = await resource.getMany(['user-1', 'user-2', 'user-3']);
|
|
10275
|
+
* users.forEach(user => console.log(user.name));
|
|
10276
|
+
*/
|
|
10277
|
+
async getMany(ids) {
|
|
10278
|
+
const { results, errors } = await PromisePool.for(ids).withConcurrency(this.client.parallelism).handleError(async (error, id) => {
|
|
10279
|
+
console.warn(`Failed to get resource ${id}:`, error.message);
|
|
10280
|
+
return {
|
|
10281
|
+
id,
|
|
10282
|
+
_error: error.message,
|
|
10283
|
+
_decryptionFailed: error.message.includes("Cipher job failed") || error.message.includes("OperationError")
|
|
10284
|
+
};
|
|
10285
|
+
}).process(async (id) => {
|
|
10286
|
+
this.emit("id", id);
|
|
10287
|
+
try {
|
|
10288
|
+
const data = await this.get(id);
|
|
10289
|
+
this.emit("data", data);
|
|
10290
|
+
return data;
|
|
10291
|
+
} catch (error) {
|
|
10292
|
+
if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
|
|
10293
|
+
console.warn(`Decryption failed for ${id}, returning basic info`);
|
|
10294
|
+
return {
|
|
10295
|
+
id,
|
|
10296
|
+
_decryptionFailed: true,
|
|
10297
|
+
_error: error.message
|
|
10298
|
+
};
|
|
10299
|
+
}
|
|
10300
|
+
throw error;
|
|
10301
|
+
}
|
|
10302
|
+
});
|
|
10303
|
+
this.emit("getMany", ids.length);
|
|
10304
|
+
return results;
|
|
10305
|
+
}
|
|
10306
|
+
/**
|
|
10307
|
+
* Get all resources (equivalent to list() without pagination)
|
|
10308
|
+
* @returns {Promise<Object[]>} Array of all resource objects
|
|
10309
|
+
* @example
|
|
10310
|
+
* const allUsers = await resource.getAll();
|
|
10311
|
+
* console.log(`Total users: ${allUsers.length}`);
|
|
10312
|
+
*/
|
|
10313
|
+
async getAll() {
|
|
10060
10314
|
let ids = await this.listIds();
|
|
10061
10315
|
if (ids.length === 0) return [];
|
|
10062
10316
|
const { results, errors } = await PromisePool.for(ids).withConcurrency(this.client.parallelism).handleError(async (error, id) => {
|
|
@@ -10358,21 +10612,14 @@ ${validation.errors.join("\n")}`);
|
|
|
10358
10612
|
for (const [partitionName, partition] of Object.entries(partitions)) {
|
|
10359
10613
|
const partitionKey = this.getPartitionKey({ partitionName, id: data.id, data });
|
|
10360
10614
|
if (partitionKey) {
|
|
10361
|
-
const mappedData = await this.schema.mapper(data);
|
|
10362
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10363
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleInsert({
|
|
10364
|
-
resource: this,
|
|
10365
|
-
data,
|
|
10366
|
-
mappedData
|
|
10367
|
-
});
|
|
10368
10615
|
const partitionMetadata = {
|
|
10369
|
-
|
|
10370
|
-
_version: this.version
|
|
10616
|
+
_v: String(this.version)
|
|
10371
10617
|
};
|
|
10372
10618
|
await this.client.putObject({
|
|
10373
10619
|
key: partitionKey,
|
|
10374
10620
|
metadata: partitionMetadata,
|
|
10375
|
-
body
|
|
10621
|
+
body: "",
|
|
10622
|
+
contentType: void 0
|
|
10376
10623
|
});
|
|
10377
10624
|
}
|
|
10378
10625
|
}
|
|
@@ -10529,25 +10776,14 @@ ${validation.errors.join("\n")}`);
|
|
|
10529
10776
|
}
|
|
10530
10777
|
if (newPartitionKey) {
|
|
10531
10778
|
try {
|
|
10532
|
-
const mappedData = await this.schema.mapper(newData);
|
|
10533
|
-
if (mappedData.undefined !== void 0) delete mappedData.undefined;
|
|
10534
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10535
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
10536
|
-
resource: this,
|
|
10537
|
-
id,
|
|
10538
|
-
data: newData,
|
|
10539
|
-
mappedData
|
|
10540
|
-
});
|
|
10541
|
-
if (processedMetadata.undefined !== void 0) delete processedMetadata.undefined;
|
|
10542
10779
|
const partitionMetadata = {
|
|
10543
|
-
|
|
10544
|
-
_version: this.version
|
|
10780
|
+
_v: String(this.version)
|
|
10545
10781
|
};
|
|
10546
|
-
if (partitionMetadata.undefined !== void 0) delete partitionMetadata.undefined;
|
|
10547
10782
|
await this.client.putObject({
|
|
10548
10783
|
key: newPartitionKey,
|
|
10549
10784
|
metadata: partitionMetadata,
|
|
10550
|
-
body
|
|
10785
|
+
body: "",
|
|
10786
|
+
contentType: void 0
|
|
10551
10787
|
});
|
|
10552
10788
|
} catch (error) {
|
|
10553
10789
|
console.warn(`New partition object could not be created for ${partitionName}:`, error.message);
|
|
@@ -10555,25 +10791,14 @@ ${validation.errors.join("\n")}`);
|
|
|
10555
10791
|
}
|
|
10556
10792
|
} else if (newPartitionKey) {
|
|
10557
10793
|
try {
|
|
10558
|
-
const mappedData = await this.schema.mapper(newData);
|
|
10559
|
-
if (mappedData.undefined !== void 0) delete mappedData.undefined;
|
|
10560
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10561
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
10562
|
-
resource: this,
|
|
10563
|
-
id,
|
|
10564
|
-
data: newData,
|
|
10565
|
-
mappedData
|
|
10566
|
-
});
|
|
10567
|
-
if (processedMetadata.undefined !== void 0) delete processedMetadata.undefined;
|
|
10568
10794
|
const partitionMetadata = {
|
|
10569
|
-
|
|
10570
|
-
_version: this.version
|
|
10795
|
+
_v: String(this.version)
|
|
10571
10796
|
};
|
|
10572
|
-
if (partitionMetadata.undefined !== void 0) delete partitionMetadata.undefined;
|
|
10573
10797
|
await this.client.putObject({
|
|
10574
10798
|
key: newPartitionKey,
|
|
10575
10799
|
metadata: partitionMetadata,
|
|
10576
|
-
body
|
|
10800
|
+
body: "",
|
|
10801
|
+
contentType: void 0
|
|
10577
10802
|
});
|
|
10578
10803
|
} catch (error) {
|
|
10579
10804
|
console.warn(`Partition object could not be updated for ${partitionName}:`, error.message);
|
|
@@ -10596,23 +10821,15 @@ ${validation.errors.join("\n")}`);
|
|
|
10596
10821
|
}
|
|
10597
10822
|
const partitionKey = this.getPartitionKey({ partitionName, id: data.id, data });
|
|
10598
10823
|
if (partitionKey) {
|
|
10599
|
-
const mappedData = await this.schema.mapper(data);
|
|
10600
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10601
|
-
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
10602
|
-
resource: this,
|
|
10603
|
-
id: data.id,
|
|
10604
|
-
data,
|
|
10605
|
-
mappedData
|
|
10606
|
-
});
|
|
10607
10824
|
const partitionMetadata = {
|
|
10608
|
-
|
|
10609
|
-
_version: this.version
|
|
10825
|
+
_v: String(this.version)
|
|
10610
10826
|
};
|
|
10611
10827
|
try {
|
|
10612
10828
|
await this.client.putObject({
|
|
10613
10829
|
key: partitionKey,
|
|
10614
10830
|
metadata: partitionMetadata,
|
|
10615
|
-
body
|
|
10831
|
+
body: "",
|
|
10832
|
+
contentType: void 0
|
|
10616
10833
|
});
|
|
10617
10834
|
} catch (error) {
|
|
10618
10835
|
console.warn(`Partition object could not be updated for ${partitionName}:`, error.message);
|
|
@@ -10662,40 +10879,75 @@ ${validation.errors.join("\n")}`);
|
|
|
10662
10879
|
throw new Error(`No partition values provided for partition '${partitionName}'`);
|
|
10663
10880
|
}
|
|
10664
10881
|
const partitionKey = join(`resource=${this.name}`, `partition=${partitionName}`, ...partitionSegments, `id=${id}`);
|
|
10665
|
-
|
|
10666
|
-
|
|
10667
|
-
|
|
10668
|
-
|
|
10669
|
-
const behaviorImpl = getBehavior(this.behavior);
|
|
10670
|
-
let body = "";
|
|
10671
|
-
if (request.ContentLength > 0) {
|
|
10672
|
-
try {
|
|
10673
|
-
const fullObject = await this.client.getObject(partitionKey);
|
|
10674
|
-
body = await streamToString(fullObject.Body);
|
|
10675
|
-
} catch (error) {
|
|
10676
|
-
body = "";
|
|
10677
|
-
}
|
|
10882
|
+
try {
|
|
10883
|
+
await this.client.headObject(partitionKey);
|
|
10884
|
+
} catch (error) {
|
|
10885
|
+
throw new Error(`Resource with id '${id}' not found in partition '${partitionName}'`);
|
|
10678
10886
|
}
|
|
10679
|
-
const
|
|
10680
|
-
resource: this,
|
|
10681
|
-
metadata,
|
|
10682
|
-
body
|
|
10683
|
-
});
|
|
10684
|
-
let data = processedMetadata;
|
|
10685
|
-
data.id = id;
|
|
10686
|
-
data._contentLength = request.ContentLength;
|
|
10687
|
-
data._lastModified = request.LastModified;
|
|
10688
|
-
data._hasContent = request.ContentLength > 0;
|
|
10689
|
-
data._mimeType = request.ContentType || null;
|
|
10887
|
+
const data = await this.get(id);
|
|
10690
10888
|
data._partition = partitionName;
|
|
10691
10889
|
data._partitionValues = partitionValues;
|
|
10692
|
-
if (request.VersionId) data._versionId = request.VersionId;
|
|
10693
|
-
if (request.Expiration) data._expiresAt = request.Expiration;
|
|
10694
|
-
data._definitionHash = this.getDefinitionHash();
|
|
10695
|
-
if (data.undefined !== void 0) delete data.undefined;
|
|
10696
10890
|
this.emit("getFromPartition", data);
|
|
10697
10891
|
return data;
|
|
10698
10892
|
}
|
|
10893
|
+
/**
|
|
10894
|
+
* Create a historical version of an object
|
|
10895
|
+
* @param {string} id - Resource ID
|
|
10896
|
+
* @param {Object} data - Object data to store historically
|
|
10897
|
+
*/
|
|
10898
|
+
async createHistoricalVersion(id, data) {
|
|
10899
|
+
const historicalKey = join(`resource=${this.name}`, `historical`, `id=${id}`);
|
|
10900
|
+
const historicalData = {
|
|
10901
|
+
...data,
|
|
10902
|
+
_v: data._v || this.version,
|
|
10903
|
+
_historicalTimestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
10904
|
+
};
|
|
10905
|
+
const mappedData = await this.schema.mapper(historicalData);
|
|
10906
|
+
const behaviorImpl = getBehavior(this.behavior);
|
|
10907
|
+
const { mappedData: processedMetadata, body } = await behaviorImpl.handleInsert({
|
|
10908
|
+
resource: this,
|
|
10909
|
+
data: historicalData,
|
|
10910
|
+
mappedData
|
|
10911
|
+
});
|
|
10912
|
+
const finalMetadata = {
|
|
10913
|
+
...processedMetadata,
|
|
10914
|
+
_v: data._v || this.version,
|
|
10915
|
+
_historicalTimestamp: historicalData._historicalTimestamp
|
|
10916
|
+
};
|
|
10917
|
+
let contentType = void 0;
|
|
10918
|
+
if (body && body !== "") {
|
|
10919
|
+
try {
|
|
10920
|
+
JSON.parse(body);
|
|
10921
|
+
contentType = "application/json";
|
|
10922
|
+
} catch {
|
|
10923
|
+
}
|
|
10924
|
+
}
|
|
10925
|
+
await this.client.putObject({
|
|
10926
|
+
key: historicalKey,
|
|
10927
|
+
metadata: finalMetadata,
|
|
10928
|
+
body,
|
|
10929
|
+
contentType
|
|
10930
|
+
});
|
|
10931
|
+
}
|
|
10932
|
+
/**
|
|
10933
|
+
* Apply version mapping to convert an object from one version to another
|
|
10934
|
+
* @param {Object} data - Object data to map
|
|
10935
|
+
* @param {string} fromVersion - Source version
|
|
10936
|
+
* @param {string} toVersion - Target version
|
|
10937
|
+
* @returns {Object} Mapped object data
|
|
10938
|
+
*/
|
|
10939
|
+
async applyVersionMapping(data, fromVersion, toVersion) {
|
|
10940
|
+
if (fromVersion === toVersion) {
|
|
10941
|
+
return data;
|
|
10942
|
+
}
|
|
10943
|
+
const mappedData = {
|
|
10944
|
+
...data,
|
|
10945
|
+
_v: toVersion,
|
|
10946
|
+
_originalVersion: fromVersion,
|
|
10947
|
+
_versionMapped: true
|
|
10948
|
+
};
|
|
10949
|
+
return mappedData;
|
|
10950
|
+
}
|
|
10699
10951
|
}
|
|
10700
10952
|
function validateResourceConfig(config) {
|
|
10701
10953
|
const errors = [];
|
|
@@ -10741,6 +10993,20 @@ function validateResourceConfig(config) {
|
|
|
10741
10993
|
errors.push(`Resource '${field}' must be a boolean`);
|
|
10742
10994
|
}
|
|
10743
10995
|
}
|
|
10996
|
+
if (config.idGenerator !== void 0) {
|
|
10997
|
+
if (typeof config.idGenerator !== "function" && typeof config.idGenerator !== "number") {
|
|
10998
|
+
errors.push("Resource 'idGenerator' must be a function or a number (size)");
|
|
10999
|
+
} else if (typeof config.idGenerator === "number" && config.idGenerator <= 0) {
|
|
11000
|
+
errors.push("Resource 'idGenerator' size must be greater than 0");
|
|
11001
|
+
}
|
|
11002
|
+
}
|
|
11003
|
+
if (config.idSize !== void 0) {
|
|
11004
|
+
if (typeof config.idSize !== "number" || !Number.isInteger(config.idSize)) {
|
|
11005
|
+
errors.push("Resource 'idSize' must be an integer");
|
|
11006
|
+
} else if (config.idSize <= 0) {
|
|
11007
|
+
errors.push("Resource 'idSize' must be greater than 0");
|
|
11008
|
+
}
|
|
11009
|
+
}
|
|
10744
11010
|
if (config.partitions !== void 0) {
|
|
10745
11011
|
if (typeof config.partitions !== "object" || Array.isArray(config.partitions)) {
|
|
10746
11012
|
errors.push("Resource 'partitions' must be an object");
|
|
@@ -10796,7 +11062,7 @@ class Database extends EventEmitter {
|
|
|
10796
11062
|
this.version = "1";
|
|
10797
11063
|
this.s3dbVersion = (() => {
|
|
10798
11064
|
try {
|
|
10799
|
-
return true ? "
|
|
11065
|
+
return true ? "6.0.0" : "latest";
|
|
10800
11066
|
} catch (e) {
|
|
10801
11067
|
return "latest";
|
|
10802
11068
|
}
|
|
@@ -10807,8 +11073,10 @@ class Database extends EventEmitter {
|
|
|
10807
11073
|
this.verbose = options.verbose || false;
|
|
10808
11074
|
this.parallelism = parseInt(options.parallelism + "") || 10;
|
|
10809
11075
|
this.plugins = options.plugins || [];
|
|
11076
|
+
this.pluginList = options.plugins || [];
|
|
10810
11077
|
this.cache = options.cache;
|
|
10811
11078
|
this.passphrase = options.passphrase || "secret";
|
|
11079
|
+
this.versioningEnabled = options.versioningEnabled || false;
|
|
10812
11080
|
let connectionString = options.connectionString;
|
|
10813
11081
|
if (!connectionString && (options.bucket || options.accessKeyId || options.secretAccessKey)) {
|
|
10814
11082
|
const { bucket, region, accessKeyId, secretAccessKey, endpoint, forcePathStyle } = options;
|
|
@@ -10859,7 +11127,7 @@ class Database extends EventEmitter {
|
|
|
10859
11127
|
client: this.client,
|
|
10860
11128
|
version: currentVersion,
|
|
10861
11129
|
attributes: versionData.attributes,
|
|
10862
|
-
behavior: versionData.behavior || "user-
|
|
11130
|
+
behavior: versionData.behavior || "user-managed",
|
|
10863
11131
|
parallelism: this.parallelism,
|
|
10864
11132
|
passphrase: this.passphrase,
|
|
10865
11133
|
observers: [this],
|
|
@@ -10869,7 +11137,8 @@ class Database extends EventEmitter {
|
|
|
10869
11137
|
paranoid: versionData.paranoid !== void 0 ? versionData.paranoid : true,
|
|
10870
11138
|
allNestedObjectsOptional: versionData.allNestedObjectsOptional !== void 0 ? versionData.allNestedObjectsOptional : true,
|
|
10871
11139
|
autoDecrypt: versionData.autoDecrypt !== void 0 ? versionData.autoDecrypt : true,
|
|
10872
|
-
hooks: versionData.hooks || {}
|
|
11140
|
+
hooks: versionData.hooks || {},
|
|
11141
|
+
versioningEnabled: this.versioningEnabled
|
|
10873
11142
|
});
|
|
10874
11143
|
}
|
|
10875
11144
|
}
|
|
@@ -10944,7 +11213,7 @@ class Database extends EventEmitter {
|
|
|
10944
11213
|
}
|
|
10945
11214
|
const hashObj = {
|
|
10946
11215
|
attributes: stableAttributes,
|
|
10947
|
-
behavior: behavior || definition.behavior || "user-
|
|
11216
|
+
behavior: behavior || definition.behavior || "user-managed"
|
|
10948
11217
|
};
|
|
10949
11218
|
const stableString = jsonStableStringify(hashObj);
|
|
10950
11219
|
return `sha256:${createHash("sha256").update(stableString).digest("hex")}`;
|
|
@@ -10961,8 +11230,8 @@ class Database extends EventEmitter {
|
|
|
10961
11230
|
}
|
|
10962
11231
|
async startPlugins() {
|
|
10963
11232
|
const db = this;
|
|
10964
|
-
if (!isEmpty(this.
|
|
10965
|
-
const plugins = this.
|
|
11233
|
+
if (!isEmpty(this.pluginList)) {
|
|
11234
|
+
const plugins = this.pluginList.map((p) => isFunction$1(p) ? new p(this) : p);
|
|
10966
11235
|
const setupProms = plugins.map(async (plugin) => {
|
|
10967
11236
|
if (plugin.beforeSetup) await plugin.beforeSetup();
|
|
10968
11237
|
await plugin.setup(db);
|
|
@@ -10977,6 +11246,20 @@ class Database extends EventEmitter {
|
|
|
10977
11246
|
await Promise.all(startProms);
|
|
10978
11247
|
}
|
|
10979
11248
|
}
|
|
11249
|
+
/**
|
|
11250
|
+
* Register and setup a plugin
|
|
11251
|
+
* @param {Plugin} plugin - Plugin instance to register
|
|
11252
|
+
* @param {string} [name] - Optional name for the plugin (defaults to plugin.constructor.name)
|
|
11253
|
+
*/
|
|
11254
|
+
async usePlugin(plugin, name = null) {
|
|
11255
|
+
const pluginName = name || plugin.constructor.name.replace("Plugin", "").toLowerCase();
|
|
11256
|
+
this.plugins[pluginName] = plugin;
|
|
11257
|
+
if (this.isConnected()) {
|
|
11258
|
+
await plugin.setup(this);
|
|
11259
|
+
await plugin.start();
|
|
11260
|
+
}
|
|
11261
|
+
return plugin;
|
|
11262
|
+
}
|
|
10980
11263
|
async uploadMetadataFile() {
|
|
10981
11264
|
const metadata = {
|
|
10982
11265
|
version: this.version,
|
|
@@ -11007,7 +11290,7 @@ class Database extends EventEmitter {
|
|
|
11007
11290
|
[version]: {
|
|
11008
11291
|
hash: definitionHash,
|
|
11009
11292
|
attributes: resourceDef.attributes,
|
|
11010
|
-
behavior: resourceDef.behavior || "user-
|
|
11293
|
+
behavior: resourceDef.behavior || "user-managed",
|
|
11011
11294
|
timestamps: resource.config.timestamps,
|
|
11012
11295
|
partitions: resource.config.partitions,
|
|
11013
11296
|
paranoid: resource.config.paranoid,
|
|
@@ -11056,7 +11339,7 @@ class Database extends EventEmitter {
|
|
|
11056
11339
|
* @param {Object} [config.options] - Resource options (deprecated, use root level parameters)
|
|
11057
11340
|
* @returns {Object} Result with exists and hash information
|
|
11058
11341
|
*/
|
|
11059
|
-
resourceExistsWithSameHash({ name, attributes, behavior = "user-
|
|
11342
|
+
resourceExistsWithSameHash({ name, attributes, behavior = "user-managed", options = {} }) {
|
|
11060
11343
|
if (!this.resources[name]) {
|
|
11061
11344
|
return { exists: false, sameHash: false, hash: null };
|
|
11062
11345
|
}
|
|
@@ -11069,6 +11352,7 @@ class Database extends EventEmitter {
|
|
|
11069
11352
|
client: this.client,
|
|
11070
11353
|
version: existingResource.version,
|
|
11071
11354
|
passphrase: this.passphrase,
|
|
11355
|
+
versioningEnabled: this.versioningEnabled,
|
|
11072
11356
|
...options
|
|
11073
11357
|
});
|
|
11074
11358
|
const newHash = this.generateDefinitionHash(mockResource.export());
|
|
@@ -11079,7 +11363,7 @@ class Database extends EventEmitter {
|
|
|
11079
11363
|
existingHash
|
|
11080
11364
|
};
|
|
11081
11365
|
}
|
|
11082
|
-
async createResource({ name, attributes, behavior = "user-
|
|
11366
|
+
async createResource({ name, attributes, behavior = "user-managed", hooks, ...config }) {
|
|
11083
11367
|
if (this.resources[name]) {
|
|
11084
11368
|
const existingResource = this.resources[name];
|
|
11085
11369
|
Object.assign(existingResource.config, {
|
|
@@ -11089,6 +11373,7 @@ class Database extends EventEmitter {
|
|
|
11089
11373
|
if (behavior) {
|
|
11090
11374
|
existingResource.behavior = behavior;
|
|
11091
11375
|
}
|
|
11376
|
+
existingResource.versioningEnabled = this.versioningEnabled;
|
|
11092
11377
|
existingResource.updateAttributes(attributes);
|
|
11093
11378
|
if (hooks) {
|
|
11094
11379
|
for (const [event, hooksArr] of Object.entries(hooks)) {
|
|
@@ -11123,6 +11408,7 @@ class Database extends EventEmitter {
|
|
|
11123
11408
|
passphrase: this.passphrase,
|
|
11124
11409
|
cache: this.cache,
|
|
11125
11410
|
hooks,
|
|
11411
|
+
versioningEnabled: this.versioningEnabled,
|
|
11126
11412
|
...config
|
|
11127
11413
|
});
|
|
11128
11414
|
this.resources[name] = resource;
|
|
@@ -17589,12 +17875,117 @@ class S3Cache extends Cache {
|
|
|
17589
17875
|
}
|
|
17590
17876
|
|
|
17591
17877
|
class Plugin extends EventEmitter {
|
|
17878
|
+
constructor(options = {}) {
|
|
17879
|
+
super();
|
|
17880
|
+
this.name = this.constructor.name;
|
|
17881
|
+
this.options = options;
|
|
17882
|
+
this.hooks = /* @__PURE__ */ new Map();
|
|
17883
|
+
}
|
|
17592
17884
|
async setup(database) {
|
|
17885
|
+
this.database = database;
|
|
17886
|
+
this.beforeSetup();
|
|
17887
|
+
await this.onSetup();
|
|
17888
|
+
this.afterSetup();
|
|
17593
17889
|
}
|
|
17594
17890
|
async start() {
|
|
17891
|
+
this.beforeStart();
|
|
17892
|
+
await this.onStart();
|
|
17893
|
+
this.afterStart();
|
|
17595
17894
|
}
|
|
17596
17895
|
async stop() {
|
|
17896
|
+
this.beforeStop();
|
|
17897
|
+
await this.onStop();
|
|
17898
|
+
this.afterStop();
|
|
17899
|
+
}
|
|
17900
|
+
// Override these methods in subclasses
|
|
17901
|
+
async onSetup() {
|
|
17902
|
+
}
|
|
17903
|
+
async onStart() {
|
|
17904
|
+
}
|
|
17905
|
+
async onStop() {
|
|
17906
|
+
}
|
|
17907
|
+
// Hook management methods
|
|
17908
|
+
addHook(resource, event, handler) {
|
|
17909
|
+
if (!this.hooks.has(resource)) {
|
|
17910
|
+
this.hooks.set(resource, /* @__PURE__ */ new Map());
|
|
17911
|
+
}
|
|
17912
|
+
const resourceHooks = this.hooks.get(resource);
|
|
17913
|
+
if (!resourceHooks.has(event)) {
|
|
17914
|
+
resourceHooks.set(event, []);
|
|
17915
|
+
}
|
|
17916
|
+
resourceHooks.get(event).push(handler);
|
|
17917
|
+
}
|
|
17918
|
+
removeHook(resource, event, handler) {
|
|
17919
|
+
const resourceHooks = this.hooks.get(resource);
|
|
17920
|
+
if (resourceHooks && resourceHooks.has(event)) {
|
|
17921
|
+
const handlers = resourceHooks.get(event);
|
|
17922
|
+
const index = handlers.indexOf(handler);
|
|
17923
|
+
if (index > -1) {
|
|
17924
|
+
handlers.splice(index, 1);
|
|
17925
|
+
}
|
|
17926
|
+
}
|
|
17927
|
+
}
|
|
17928
|
+
// Enhanced resource method wrapping that supports multiple plugins
|
|
17929
|
+
wrapResourceMethod(resource, methodName, wrapper) {
|
|
17930
|
+
const originalMethod = resource[methodName];
|
|
17931
|
+
if (!resource._pluginWrappers) {
|
|
17932
|
+
resource._pluginWrappers = /* @__PURE__ */ new Map();
|
|
17933
|
+
}
|
|
17934
|
+
if (!resource._pluginWrappers.has(methodName)) {
|
|
17935
|
+
resource._pluginWrappers.set(methodName, []);
|
|
17936
|
+
}
|
|
17937
|
+
resource._pluginWrappers.get(methodName).push(wrapper);
|
|
17938
|
+
if (!resource[`_wrapped_${methodName}`]) {
|
|
17939
|
+
resource[`_wrapped_${methodName}`] = originalMethod;
|
|
17940
|
+
const isJestMock = originalMethod && originalMethod._isMockFunction;
|
|
17941
|
+
resource[methodName] = async function(...args) {
|
|
17942
|
+
let result = await resource[`_wrapped_${methodName}`](...args);
|
|
17943
|
+
for (const wrapper2 of resource._pluginWrappers.get(methodName)) {
|
|
17944
|
+
result = await wrapper2.call(this, result, args, methodName);
|
|
17945
|
+
}
|
|
17946
|
+
return result;
|
|
17947
|
+
};
|
|
17948
|
+
if (isJestMock) {
|
|
17949
|
+
Object.setPrototypeOf(resource[methodName], Object.getPrototypeOf(originalMethod));
|
|
17950
|
+
Object.assign(resource[methodName], originalMethod);
|
|
17951
|
+
}
|
|
17952
|
+
}
|
|
17953
|
+
}
|
|
17954
|
+
// Partition-aware helper methods
|
|
17955
|
+
getPartitionValues(data, resource) {
|
|
17956
|
+
if (!resource.config?.partitions) return {};
|
|
17957
|
+
const partitionValues = {};
|
|
17958
|
+
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
17959
|
+
if (partitionDef.fields) {
|
|
17960
|
+
partitionValues[partitionName] = {};
|
|
17961
|
+
for (const [fieldName, rule] of Object.entries(partitionDef.fields)) {
|
|
17962
|
+
const value = this.getNestedFieldValue(data, fieldName);
|
|
17963
|
+
if (value !== null && value !== void 0) {
|
|
17964
|
+
partitionValues[partitionName][fieldName] = resource.applyPartitionRule(value, rule);
|
|
17965
|
+
}
|
|
17966
|
+
}
|
|
17967
|
+
} else {
|
|
17968
|
+
partitionValues[partitionName] = {};
|
|
17969
|
+
}
|
|
17970
|
+
}
|
|
17971
|
+
return partitionValues;
|
|
17972
|
+
}
|
|
17973
|
+
getNestedFieldValue(data, fieldPath) {
|
|
17974
|
+
if (!fieldPath.includes(".")) {
|
|
17975
|
+
return data[fieldPath] ?? null;
|
|
17976
|
+
}
|
|
17977
|
+
const keys = fieldPath.split(".");
|
|
17978
|
+
let value = data;
|
|
17979
|
+
for (const key of keys) {
|
|
17980
|
+
if (value && typeof value === "object" && key in value) {
|
|
17981
|
+
value = value[key];
|
|
17982
|
+
} else {
|
|
17983
|
+
return null;
|
|
17984
|
+
}
|
|
17985
|
+
}
|
|
17986
|
+
return value ?? null;
|
|
17597
17987
|
}
|
|
17988
|
+
// Event emission methods
|
|
17598
17989
|
beforeSetup() {
|
|
17599
17990
|
this.emit("plugin.beforeSetup", /* @__PURE__ */ new Date());
|
|
17600
17991
|
}
|
|
@@ -17624,205 +18015,2867 @@ const PluginObject = {
|
|
|
17624
18015
|
}
|
|
17625
18016
|
};
|
|
17626
18017
|
|
|
17627
|
-
|
|
17628
|
-
|
|
17629
|
-
|
|
17630
|
-
this.
|
|
17631
|
-
|
|
17632
|
-
|
|
17633
|
-
|
|
17634
|
-
|
|
17635
|
-
|
|
17636
|
-
|
|
18018
|
+
class AuditPlugin extends Plugin {
|
|
18019
|
+
constructor(options = {}) {
|
|
18020
|
+
super(options);
|
|
18021
|
+
this.auditResource = null;
|
|
18022
|
+
this.config = {
|
|
18023
|
+
enabled: options.enabled !== false,
|
|
18024
|
+
includeData: options.includeData !== false,
|
|
18025
|
+
includePartitions: options.includePartitions !== false,
|
|
18026
|
+
maxDataSize: options.maxDataSize || 1e4,
|
|
18027
|
+
// 10KB limit
|
|
18028
|
+
...options
|
|
17637
18029
|
};
|
|
17638
|
-
|
|
17639
|
-
|
|
17640
|
-
|
|
17641
|
-
|
|
17642
|
-
|
|
17643
|
-
|
|
17644
|
-
|
|
17645
|
-
|
|
17646
|
-
|
|
17647
|
-
|
|
17648
|
-
|
|
17649
|
-
|
|
17650
|
-
|
|
17651
|
-
|
|
17652
|
-
|
|
17653
|
-
|
|
17654
|
-
|
|
17655
|
-
|
|
17656
|
-
|
|
17657
|
-
|
|
17658
|
-
|
|
17659
|
-
|
|
17660
|
-
|
|
17661
|
-
|
|
17662
|
-
|
|
17663
|
-
|
|
17664
|
-
|
|
17665
|
-
|
|
17666
|
-
|
|
18030
|
+
}
|
|
18031
|
+
async onSetup() {
|
|
18032
|
+
if (!this.config.enabled) {
|
|
18033
|
+
this.auditResource = null;
|
|
18034
|
+
return;
|
|
18035
|
+
}
|
|
18036
|
+
try {
|
|
18037
|
+
this.auditResource = await this.database.createResource({
|
|
18038
|
+
name: "audits",
|
|
18039
|
+
attributes: {
|
|
18040
|
+
id: "string|required",
|
|
18041
|
+
resourceName: "string|required",
|
|
18042
|
+
operation: "string|required",
|
|
18043
|
+
recordId: "string|required",
|
|
18044
|
+
userId: "string|optional",
|
|
18045
|
+
timestamp: "string|required",
|
|
18046
|
+
oldData: "string|optional",
|
|
18047
|
+
newData: "string|optional",
|
|
18048
|
+
partition: "string|optional",
|
|
18049
|
+
partitionValues: "string|optional",
|
|
18050
|
+
metadata: "string|optional"
|
|
18051
|
+
}
|
|
18052
|
+
});
|
|
18053
|
+
} catch (error) {
|
|
18054
|
+
try {
|
|
18055
|
+
this.auditResource = this.database.resources.audits;
|
|
18056
|
+
} catch (innerError) {
|
|
18057
|
+
this.auditResource = null;
|
|
18058
|
+
return;
|
|
18059
|
+
}
|
|
18060
|
+
}
|
|
18061
|
+
this.installDatabaseProxy();
|
|
18062
|
+
this.installResourceHooks();
|
|
18063
|
+
}
|
|
18064
|
+
async onStart() {
|
|
18065
|
+
}
|
|
18066
|
+
async onStop() {
|
|
18067
|
+
}
|
|
18068
|
+
installDatabaseProxy() {
|
|
18069
|
+
if (this.database._auditProxyInstalled) {
|
|
18070
|
+
return;
|
|
18071
|
+
}
|
|
18072
|
+
const installResourceHooksForResource = this.installResourceHooksForResource.bind(this);
|
|
18073
|
+
this.database._originalCreateResource = this.database.createResource;
|
|
18074
|
+
this.database.createResource = async function(...args) {
|
|
18075
|
+
const resource = await this._originalCreateResource(...args);
|
|
18076
|
+
if (resource.name !== "audit_logs") {
|
|
18077
|
+
installResourceHooksForResource(resource);
|
|
17667
18078
|
}
|
|
18079
|
+
return resource;
|
|
17668
18080
|
};
|
|
17669
|
-
this.
|
|
17670
|
-
}
|
|
17671
|
-
|
|
17672
|
-
|
|
17673
|
-
|
|
17674
|
-
|
|
17675
|
-
|
|
17676
|
-
|
|
17677
|
-
|
|
17678
|
-
this.
|
|
17679
|
-
|
|
17680
|
-
|
|
17681
|
-
|
|
17682
|
-
|
|
17683
|
-
|
|
17684
|
-
|
|
18081
|
+
this.database._auditProxyInstalled = true;
|
|
18082
|
+
}
|
|
18083
|
+
installResourceHooks() {
|
|
18084
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18085
|
+
if (resource.name === "audits") continue;
|
|
18086
|
+
this.installResourceHooksForResource(resource);
|
|
18087
|
+
}
|
|
18088
|
+
}
|
|
18089
|
+
installResourceHooksForResource(resource) {
|
|
18090
|
+
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
18091
|
+
const [data] = args;
|
|
18092
|
+
const recordId = data.id || result.id || "auto-generated";
|
|
18093
|
+
const partitionValues = this.config.includePartitions ? this.getPartitionValues(data, resource) : null;
|
|
18094
|
+
const auditRecord = {
|
|
18095
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18096
|
+
resourceName: resource.name,
|
|
18097
|
+
operation: "insert",
|
|
18098
|
+
recordId,
|
|
18099
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18100
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18101
|
+
oldData: null,
|
|
18102
|
+
newData: this.config.includeData === false ? null : JSON.stringify(this.truncateData(data)),
|
|
18103
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18104
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18105
|
+
metadata: JSON.stringify({
|
|
18106
|
+
source: "audit-plugin",
|
|
18107
|
+
version: "2.0"
|
|
18108
|
+
})
|
|
18109
|
+
};
|
|
18110
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18111
|
+
return result;
|
|
18112
|
+
});
|
|
18113
|
+
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
18114
|
+
const [id, data] = args;
|
|
18115
|
+
let oldData = null;
|
|
18116
|
+
if (this.config.includeData) {
|
|
18117
|
+
try {
|
|
18118
|
+
oldData = await resource.get(id);
|
|
18119
|
+
} catch (error) {
|
|
18120
|
+
}
|
|
18121
|
+
}
|
|
18122
|
+
const partitionValues = this.config.includePartitions ? this.getPartitionValues(result, resource) : null;
|
|
18123
|
+
const auditRecord = {
|
|
18124
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18125
|
+
resourceName: resource.name,
|
|
18126
|
+
operation: "update",
|
|
18127
|
+
recordId: id,
|
|
18128
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18129
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18130
|
+
oldData: oldData && this.config.includeData === false ? null : oldData ? JSON.stringify(this.truncateData(oldData)) : null,
|
|
18131
|
+
newData: this.config.includeData === false ? null : JSON.stringify(this.truncateData(result)),
|
|
18132
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18133
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18134
|
+
metadata: JSON.stringify({
|
|
18135
|
+
source: "audit-plugin",
|
|
18136
|
+
version: "2.0"
|
|
18137
|
+
})
|
|
18138
|
+
};
|
|
18139
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18140
|
+
return result;
|
|
18141
|
+
});
|
|
18142
|
+
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
18143
|
+
const [id] = args;
|
|
18144
|
+
let oldData = null;
|
|
18145
|
+
if (this.config.includeData) {
|
|
18146
|
+
try {
|
|
18147
|
+
oldData = await resource.get(id);
|
|
18148
|
+
} catch (error) {
|
|
18149
|
+
}
|
|
18150
|
+
}
|
|
18151
|
+
const partitionValues = oldData && this.config.includePartitions ? this.getPartitionValues(oldData, resource) : null;
|
|
18152
|
+
const auditRecord = {
|
|
18153
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18154
|
+
resourceName: resource.name,
|
|
18155
|
+
operation: "delete",
|
|
18156
|
+
recordId: id,
|
|
18157
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18158
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18159
|
+
oldData: oldData && this.config.includeData === false ? null : oldData ? JSON.stringify(this.truncateData(oldData)) : null,
|
|
18160
|
+
newData: null,
|
|
18161
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18162
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18163
|
+
metadata: JSON.stringify({
|
|
18164
|
+
source: "audit-plugin",
|
|
18165
|
+
version: "2.0"
|
|
18166
|
+
})
|
|
18167
|
+
};
|
|
18168
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18169
|
+
return result;
|
|
18170
|
+
});
|
|
18171
|
+
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
18172
|
+
const [ids] = args;
|
|
18173
|
+
const auditRecords = [];
|
|
18174
|
+
if (this.config.includeData) {
|
|
18175
|
+
for (const id of ids) {
|
|
18176
|
+
try {
|
|
18177
|
+
const oldData = await resource.get(id);
|
|
18178
|
+
const partitionValues = this.config.includePartitions ? this.getPartitionValues(oldData, resource) : null;
|
|
18179
|
+
auditRecords.push({
|
|
18180
|
+
id: `audit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18181
|
+
resourceName: resource.name,
|
|
18182
|
+
operation: "delete",
|
|
18183
|
+
recordId: id,
|
|
18184
|
+
userId: this.getCurrentUserId?.() || "system",
|
|
18185
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
18186
|
+
oldData: this.config.includeData === false ? null : JSON.stringify(this.truncateData(oldData)),
|
|
18187
|
+
newData: null,
|
|
18188
|
+
partition: this.config.includePartitions ? this.getPrimaryPartition(partitionValues) : null,
|
|
18189
|
+
partitionValues: this.config.includePartitions ? partitionValues ? Object.keys(partitionValues).length > 0 ? JSON.stringify(partitionValues) : null : null : null,
|
|
18190
|
+
metadata: JSON.stringify({
|
|
18191
|
+
source: "audit-plugin",
|
|
18192
|
+
version: "2.0",
|
|
18193
|
+
batchOperation: true
|
|
18194
|
+
})
|
|
18195
|
+
});
|
|
18196
|
+
} catch (error) {
|
|
18197
|
+
}
|
|
18198
|
+
}
|
|
18199
|
+
}
|
|
18200
|
+
for (const auditRecord of auditRecords) {
|
|
18201
|
+
this.logAudit(auditRecord).catch(console.error);
|
|
18202
|
+
}
|
|
18203
|
+
return result;
|
|
18204
|
+
});
|
|
17685
18205
|
}
|
|
17686
|
-
|
|
18206
|
+
getPartitionValues(data, resource) {
|
|
18207
|
+
const partitions = resource.config?.partitions || {};
|
|
18208
|
+
const partitionValues = {};
|
|
18209
|
+
for (const [partitionName, partitionDef] of Object.entries(partitions)) {
|
|
18210
|
+
if (partitionDef.fields) {
|
|
18211
|
+
const partitionData = {};
|
|
18212
|
+
for (const [fieldName, fieldRule] of Object.entries(partitionDef.fields)) {
|
|
18213
|
+
const fieldValue = this.getNestedFieldValue(data, fieldName);
|
|
18214
|
+
if (fieldValue !== void 0 && fieldValue !== null) {
|
|
18215
|
+
partitionData[fieldName] = fieldValue;
|
|
18216
|
+
}
|
|
18217
|
+
}
|
|
18218
|
+
if (Object.keys(partitionData).length > 0) {
|
|
18219
|
+
partitionValues[partitionName] = partitionData;
|
|
18220
|
+
}
|
|
18221
|
+
}
|
|
18222
|
+
}
|
|
18223
|
+
return partitionValues;
|
|
18224
|
+
}
|
|
18225
|
+
getNestedFieldValue(data, fieldPath) {
|
|
18226
|
+
if (!fieldPath.includes(".")) {
|
|
18227
|
+
return data[fieldPath];
|
|
18228
|
+
}
|
|
18229
|
+
const keys = fieldPath.split(".");
|
|
18230
|
+
let currentLevel = data;
|
|
18231
|
+
for (const key of keys) {
|
|
18232
|
+
if (!currentLevel || typeof currentLevel !== "object" || !(key in currentLevel)) {
|
|
18233
|
+
return void 0;
|
|
18234
|
+
}
|
|
18235
|
+
currentLevel = currentLevel[key];
|
|
18236
|
+
}
|
|
18237
|
+
return currentLevel;
|
|
18238
|
+
}
|
|
18239
|
+
getPrimaryPartition(partitionValues) {
|
|
18240
|
+
if (!partitionValues) return null;
|
|
18241
|
+
const partitionNames = Object.keys(partitionValues);
|
|
18242
|
+
return partitionNames.length > 0 ? partitionNames[0] : null;
|
|
18243
|
+
}
|
|
18244
|
+
async logAudit(auditRecord) {
|
|
18245
|
+
if (!this.auditResource) return;
|
|
18246
|
+
try {
|
|
18247
|
+
await this.auditResource.insert(auditRecord);
|
|
18248
|
+
} catch (error) {
|
|
18249
|
+
console.error("Failed to log audit record:", error);
|
|
18250
|
+
if (error && error.stack) console.error(error.stack);
|
|
18251
|
+
}
|
|
18252
|
+
}
|
|
18253
|
+
truncateData(data) {
|
|
18254
|
+
if (!data) return data;
|
|
18255
|
+
const dataStr = JSON.stringify(data);
|
|
18256
|
+
if (dataStr.length <= this.config.maxDataSize) {
|
|
18257
|
+
return data;
|
|
18258
|
+
}
|
|
18259
|
+
return {
|
|
18260
|
+
...data,
|
|
18261
|
+
_truncated: true,
|
|
18262
|
+
_originalSize: dataStr.length,
|
|
18263
|
+
_truncatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
18264
|
+
};
|
|
18265
|
+
}
|
|
18266
|
+
// Utility methods for querying audit logs
|
|
18267
|
+
async getAuditLogs(options = {}) {
|
|
18268
|
+
if (!this.auditResource) return [];
|
|
18269
|
+
try {
|
|
18270
|
+
const {
|
|
18271
|
+
resourceName,
|
|
18272
|
+
operation,
|
|
18273
|
+
recordId,
|
|
18274
|
+
userId,
|
|
18275
|
+
partition,
|
|
18276
|
+
startDate,
|
|
18277
|
+
endDate,
|
|
18278
|
+
limit = 100,
|
|
18279
|
+
offset = 0
|
|
18280
|
+
} = options;
|
|
18281
|
+
const allAudits = await this.auditResource.getAll();
|
|
18282
|
+
let filtered = allAudits.filter((audit) => {
|
|
18283
|
+
if (resourceName && audit.resourceName !== resourceName) return false;
|
|
18284
|
+
if (operation && audit.operation !== operation) return false;
|
|
18285
|
+
if (recordId && audit.recordId !== recordId) return false;
|
|
18286
|
+
if (userId && audit.userId !== userId) return false;
|
|
18287
|
+
if (partition && audit.partition !== partition) return false;
|
|
18288
|
+
if (startDate && new Date(audit.timestamp) < new Date(startDate)) return false;
|
|
18289
|
+
if (endDate && new Date(audit.timestamp) > new Date(endDate)) return false;
|
|
18290
|
+
return true;
|
|
18291
|
+
});
|
|
18292
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
18293
|
+
const deserialized = filtered.slice(offset, offset + limit).map((audit) => ({
|
|
18294
|
+
...audit,
|
|
18295
|
+
oldData: audit.oldData ? JSON.parse(audit.oldData) : null,
|
|
18296
|
+
newData: audit.newData ? JSON.parse(audit.newData) : null,
|
|
18297
|
+
partitionValues: audit.partitionValues ? JSON.parse(audit.partitionValues) : null,
|
|
18298
|
+
metadata: audit.metadata ? JSON.parse(audit.metadata) : null
|
|
18299
|
+
}));
|
|
18300
|
+
return deserialized;
|
|
18301
|
+
} catch (error) {
|
|
18302
|
+
console.error("Failed to get audit logs:", error);
|
|
18303
|
+
if (error && error.stack) console.error(error.stack);
|
|
18304
|
+
return [];
|
|
18305
|
+
}
|
|
18306
|
+
}
|
|
18307
|
+
async getRecordHistory(resourceName, recordId) {
|
|
18308
|
+
return this.getAuditLogs({
|
|
18309
|
+
resourceName,
|
|
18310
|
+
recordId,
|
|
18311
|
+
limit: 1e3
|
|
18312
|
+
});
|
|
18313
|
+
}
|
|
18314
|
+
async getPartitionHistory(resourceName, partitionName, partitionValues) {
|
|
18315
|
+
return this.getAuditLogs({
|
|
18316
|
+
resourceName,
|
|
18317
|
+
partition: partitionName,
|
|
18318
|
+
limit: 1e3
|
|
18319
|
+
});
|
|
18320
|
+
}
|
|
18321
|
+
async getAuditStats(options = {}) {
|
|
18322
|
+
const {
|
|
18323
|
+
resourceName,
|
|
18324
|
+
startDate,
|
|
18325
|
+
endDate
|
|
18326
|
+
} = options;
|
|
18327
|
+
const allAudits = await this.getAuditLogs({
|
|
18328
|
+
resourceName,
|
|
18329
|
+
startDate,
|
|
18330
|
+
endDate,
|
|
18331
|
+
limit: 1e4
|
|
18332
|
+
});
|
|
18333
|
+
const stats = {
|
|
18334
|
+
total: allAudits.length,
|
|
18335
|
+
byOperation: {},
|
|
18336
|
+
byResource: {},
|
|
18337
|
+
byPartition: {},
|
|
18338
|
+
byUser: {},
|
|
18339
|
+
timeline: {}
|
|
18340
|
+
};
|
|
18341
|
+
for (const audit of allAudits) {
|
|
18342
|
+
stats.byOperation[audit.operation] = (stats.byOperation[audit.operation] || 0) + 1;
|
|
18343
|
+
stats.byResource[audit.resourceName] = (stats.byResource[audit.resourceName] || 0) + 1;
|
|
18344
|
+
if (audit.partition) {
|
|
18345
|
+
stats.byPartition[audit.partition] = (stats.byPartition[audit.partition] || 0) + 1;
|
|
18346
|
+
}
|
|
18347
|
+
stats.byUser[audit.userId] = (stats.byUser[audit.userId] || 0) + 1;
|
|
18348
|
+
const day = audit.timestamp.split("T")[0];
|
|
18349
|
+
stats.timeline[day] = (stats.timeline[day] || 0) + 1;
|
|
18350
|
+
}
|
|
18351
|
+
return stats;
|
|
18352
|
+
}
|
|
18353
|
+
}
|
|
17687
18354
|
|
|
17688
18355
|
class CachePlugin extends Plugin {
|
|
17689
18356
|
constructor(options = {}) {
|
|
17690
|
-
super();
|
|
18357
|
+
super(options);
|
|
17691
18358
|
this.driver = options.driver;
|
|
18359
|
+
this.config = {
|
|
18360
|
+
enabled: options.enabled !== false,
|
|
18361
|
+
includePartitions: options.includePartitions !== false,
|
|
18362
|
+
...options
|
|
18363
|
+
};
|
|
17692
18364
|
}
|
|
17693
18365
|
async setup(database) {
|
|
17694
|
-
this.
|
|
17695
|
-
|
|
17696
|
-
keyPrefix: "cache",
|
|
17697
|
-
client: database.client
|
|
17698
|
-
});
|
|
17699
|
-
this.installDatabaseProxy();
|
|
17700
|
-
for (const resource of Object.values(database.resources)) {
|
|
17701
|
-
this.installResourcesProxies(resource);
|
|
18366
|
+
if (!this.config.enabled) {
|
|
18367
|
+
return;
|
|
17702
18368
|
}
|
|
18369
|
+
await super.setup(database);
|
|
17703
18370
|
}
|
|
17704
|
-
async
|
|
18371
|
+
async onSetup() {
|
|
18372
|
+
if (this.config.driver) {
|
|
18373
|
+
this.driver = this.config.driver;
|
|
18374
|
+
} else if (this.config.driverType === "memory") {
|
|
18375
|
+
this.driver = new MemoryCache(this.config.memoryOptions || {});
|
|
18376
|
+
} else {
|
|
18377
|
+
this.driver = new S3Cache(this.config.s3Options || {});
|
|
18378
|
+
}
|
|
18379
|
+
this.installDatabaseProxy();
|
|
18380
|
+
this.installResourceHooks();
|
|
17705
18381
|
}
|
|
17706
|
-
async
|
|
18382
|
+
async onStart() {
|
|
18383
|
+
}
|
|
18384
|
+
async onStop() {
|
|
17707
18385
|
}
|
|
17708
18386
|
installDatabaseProxy() {
|
|
17709
|
-
|
|
17710
|
-
|
|
18387
|
+
if (this.database._cacheProxyInstalled) {
|
|
18388
|
+
return;
|
|
18389
|
+
}
|
|
18390
|
+
const installResourceHooks = this.installResourceHooks.bind(this);
|
|
18391
|
+
this.database._originalCreateResourceForCache = this.database.createResource;
|
|
17711
18392
|
this.database.createResource = async function(...args) {
|
|
17712
|
-
const resource = await this.
|
|
17713
|
-
|
|
18393
|
+
const resource = await this._originalCreateResourceForCache(...args);
|
|
18394
|
+
installResourceHooks(resource);
|
|
17714
18395
|
return resource;
|
|
17715
18396
|
};
|
|
18397
|
+
this.database._cacheProxyInstalled = true;
|
|
18398
|
+
}
|
|
18399
|
+
installResourceHooks() {
|
|
18400
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18401
|
+
this.installResourceHooksForResource(resource);
|
|
18402
|
+
}
|
|
17716
18403
|
}
|
|
17717
|
-
|
|
18404
|
+
installResourceHooksForResource(resource) {
|
|
18405
|
+
if (!this.driver) return;
|
|
17718
18406
|
resource.cache = this.driver;
|
|
17719
|
-
|
|
17720
|
-
|
|
17721
|
-
|
|
17722
|
-
params = {},
|
|
17723
|
-
action = "list"
|
|
17724
|
-
}) {
|
|
17725
|
-
let key = Object.keys(params).sort().map((x) => `${x}:${params[x]}`).join("|") || "empty";
|
|
17726
|
-
key = await sha256(key);
|
|
17727
|
-
key = join(keyPrefix, `action=${action}`, `${key}.json.gz`);
|
|
17728
|
-
return key;
|
|
18407
|
+
resource.cacheKeyFor = async (options = {}) => {
|
|
18408
|
+
const { action, params = {}, partition, partitionValues } = options;
|
|
18409
|
+
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
17729
18410
|
};
|
|
17730
|
-
resource.
|
|
17731
|
-
resource.
|
|
17732
|
-
resource.
|
|
17733
|
-
resource.
|
|
17734
|
-
resource.
|
|
17735
|
-
resource.
|
|
17736
|
-
|
|
18411
|
+
resource._originalCount = resource.count;
|
|
18412
|
+
resource._originalListIds = resource.listIds;
|
|
18413
|
+
resource._originalGetMany = resource.getMany;
|
|
18414
|
+
resource._originalGetAll = resource.getAll;
|
|
18415
|
+
resource._originalPage = resource.page;
|
|
18416
|
+
resource._originalList = resource.list;
|
|
18417
|
+
resource.count = async function(options = {}) {
|
|
18418
|
+
const { partition, partitionValues } = options;
|
|
18419
|
+
const key = await resource.cacheKeyFor({
|
|
18420
|
+
action: "count",
|
|
18421
|
+
partition,
|
|
18422
|
+
partitionValues
|
|
18423
|
+
});
|
|
17737
18424
|
try {
|
|
17738
|
-
const cached = await
|
|
17739
|
-
if (cached) return cached;
|
|
18425
|
+
const cached = await resource.cache.get(key);
|
|
18426
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
17740
18427
|
} catch (err) {
|
|
17741
18428
|
if (err.name !== "NoSuchKey") throw err;
|
|
17742
18429
|
}
|
|
17743
|
-
const
|
|
17744
|
-
await
|
|
17745
|
-
return
|
|
18430
|
+
const result = await resource._originalCount(options);
|
|
18431
|
+
await resource.cache.set(key, result);
|
|
18432
|
+
return result;
|
|
17746
18433
|
};
|
|
17747
|
-
resource.listIds = async function() {
|
|
17748
|
-
const
|
|
18434
|
+
resource.listIds = async function(options = {}) {
|
|
18435
|
+
const { partition, partitionValues } = options;
|
|
18436
|
+
const key = await resource.cacheKeyFor({
|
|
18437
|
+
action: "listIds",
|
|
18438
|
+
partition,
|
|
18439
|
+
partitionValues
|
|
18440
|
+
});
|
|
17749
18441
|
try {
|
|
17750
|
-
const cached = await
|
|
17751
|
-
if (cached) return cached;
|
|
18442
|
+
const cached = await resource.cache.get(key);
|
|
18443
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
17752
18444
|
} catch (err) {
|
|
17753
18445
|
if (err.name !== "NoSuchKey") throw err;
|
|
17754
18446
|
}
|
|
17755
|
-
const
|
|
17756
|
-
await
|
|
17757
|
-
return
|
|
18447
|
+
const result = await resource._originalListIds(options);
|
|
18448
|
+
await resource.cache.set(key, result);
|
|
18449
|
+
return result;
|
|
17758
18450
|
};
|
|
17759
18451
|
resource.getMany = async function(ids) {
|
|
17760
|
-
const key = await
|
|
18452
|
+
const key = await resource.cacheKeyFor({
|
|
17761
18453
|
action: "getMany",
|
|
17762
18454
|
params: { ids }
|
|
17763
18455
|
});
|
|
17764
18456
|
try {
|
|
17765
|
-
const cached = await
|
|
17766
|
-
if (cached) return cached;
|
|
18457
|
+
const cached = await resource.cache.get(key);
|
|
18458
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
17767
18459
|
} catch (err) {
|
|
17768
18460
|
if (err.name !== "NoSuchKey") throw err;
|
|
17769
18461
|
}
|
|
17770
|
-
const
|
|
17771
|
-
await
|
|
17772
|
-
return
|
|
18462
|
+
const result = await resource._originalGetMany(ids);
|
|
18463
|
+
await resource.cache.set(key, result);
|
|
18464
|
+
return result;
|
|
17773
18465
|
};
|
|
17774
18466
|
resource.getAll = async function() {
|
|
17775
|
-
const key = await
|
|
18467
|
+
const key = await resource.cacheKeyFor({ action: "getAll" });
|
|
17776
18468
|
try {
|
|
17777
|
-
const cached = await
|
|
17778
|
-
if (cached) return cached;
|
|
18469
|
+
const cached = await resource.cache.get(key);
|
|
18470
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
17779
18471
|
} catch (err) {
|
|
17780
18472
|
if (err.name !== "NoSuchKey") throw err;
|
|
17781
18473
|
}
|
|
17782
|
-
const
|
|
17783
|
-
await
|
|
17784
|
-
return
|
|
18474
|
+
const result = await resource._originalGetAll();
|
|
18475
|
+
await resource.cache.set(key, result);
|
|
18476
|
+
return result;
|
|
17785
18477
|
};
|
|
17786
|
-
resource.page = async function({ offset, size }) {
|
|
17787
|
-
const key = await
|
|
18478
|
+
resource.page = async function({ offset, size, partition, partitionValues } = {}) {
|
|
18479
|
+
const key = await resource.cacheKeyFor({
|
|
17788
18480
|
action: "page",
|
|
17789
|
-
params: { offset, size }
|
|
18481
|
+
params: { offset, size },
|
|
18482
|
+
partition,
|
|
18483
|
+
partitionValues
|
|
17790
18484
|
});
|
|
17791
18485
|
try {
|
|
17792
|
-
const cached = await
|
|
17793
|
-
if (cached) return cached;
|
|
18486
|
+
const cached = await resource.cache.get(key);
|
|
18487
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
17794
18488
|
} catch (err) {
|
|
17795
18489
|
if (err.name !== "NoSuchKey") throw err;
|
|
17796
18490
|
}
|
|
17797
|
-
const
|
|
17798
|
-
await
|
|
17799
|
-
return
|
|
18491
|
+
const result = await resource._originalPage({ offset, size, partition, partitionValues });
|
|
18492
|
+
await resource.cache.set(key, result);
|
|
18493
|
+
return result;
|
|
17800
18494
|
};
|
|
17801
|
-
resource.
|
|
17802
|
-
|
|
17803
|
-
|
|
17804
|
-
|
|
17805
|
-
|
|
17806
|
-
|
|
17807
|
-
|
|
17808
|
-
|
|
18495
|
+
resource.list = async function(options = {}) {
|
|
18496
|
+
const { partition, partitionValues } = options;
|
|
18497
|
+
const key = await resource.cacheKeyFor({
|
|
18498
|
+
action: "list",
|
|
18499
|
+
partition,
|
|
18500
|
+
partitionValues
|
|
18501
|
+
});
|
|
18502
|
+
try {
|
|
18503
|
+
const cached = await resource.cache.get(key);
|
|
18504
|
+
if (cached !== null && cached !== void 0) return cached;
|
|
18505
|
+
} catch (err) {
|
|
18506
|
+
if (err.name !== "NoSuchKey") throw err;
|
|
18507
|
+
}
|
|
18508
|
+
const result = await resource._originalList(options);
|
|
18509
|
+
await resource.cache.set(key, result);
|
|
18510
|
+
return result;
|
|
17809
18511
|
};
|
|
17810
|
-
resource
|
|
17811
|
-
const data =
|
|
17812
|
-
await this.
|
|
17813
|
-
return
|
|
18512
|
+
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
18513
|
+
const [data] = args;
|
|
18514
|
+
await this.clearCacheForResource(resource, data);
|
|
18515
|
+
return result;
|
|
18516
|
+
});
|
|
18517
|
+
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
18518
|
+
const [id, data] = args;
|
|
18519
|
+
await this.clearCacheForResource(resource, { id, ...data });
|
|
18520
|
+
return result;
|
|
18521
|
+
});
|
|
18522
|
+
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
18523
|
+
const [id] = args;
|
|
18524
|
+
let data = { id };
|
|
18525
|
+
if (typeof resource.get === "function") {
|
|
18526
|
+
try {
|
|
18527
|
+
const full = await resource.get(id);
|
|
18528
|
+
if (full) data = full;
|
|
18529
|
+
} catch {
|
|
18530
|
+
}
|
|
18531
|
+
}
|
|
18532
|
+
await this.clearCacheForResource(resource, data);
|
|
18533
|
+
return result;
|
|
18534
|
+
});
|
|
18535
|
+
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
18536
|
+
const [ids] = args;
|
|
18537
|
+
for (const id of ids) {
|
|
18538
|
+
let data = { id };
|
|
18539
|
+
if (typeof resource.get === "function") {
|
|
18540
|
+
try {
|
|
18541
|
+
const full = await resource.get(id);
|
|
18542
|
+
if (full) data = full;
|
|
18543
|
+
} catch {
|
|
18544
|
+
}
|
|
18545
|
+
}
|
|
18546
|
+
await this.clearCacheForResource(resource, data);
|
|
18547
|
+
}
|
|
18548
|
+
return result;
|
|
18549
|
+
});
|
|
18550
|
+
}
|
|
18551
|
+
async clearCacheForResource(resource, data) {
|
|
18552
|
+
if (!resource.cache) return;
|
|
18553
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
18554
|
+
await resource.cache.clear(keyPrefix);
|
|
18555
|
+
if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
|
|
18556
|
+
const partitionValues = this.getPartitionValues(data, resource);
|
|
18557
|
+
for (const [partitionName, values] of Object.entries(partitionValues)) {
|
|
18558
|
+
if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
|
|
18559
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
18560
|
+
await resource.cache.clear(partitionKeyPrefix);
|
|
18561
|
+
}
|
|
18562
|
+
}
|
|
18563
|
+
}
|
|
18564
|
+
}
|
|
18565
|
+
async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
|
|
18566
|
+
const keyParts = [
|
|
18567
|
+
`resource=${resource.name}`,
|
|
18568
|
+
`action=${action}`
|
|
18569
|
+
];
|
|
18570
|
+
if (partition && partitionValues && Object.keys(partitionValues).length > 0) {
|
|
18571
|
+
keyParts.push(`partition:${partition}`);
|
|
18572
|
+
for (const [field, value] of Object.entries(partitionValues)) {
|
|
18573
|
+
if (value !== null && value !== void 0) {
|
|
18574
|
+
keyParts.push(`${field}:${value}`);
|
|
18575
|
+
}
|
|
18576
|
+
}
|
|
18577
|
+
}
|
|
18578
|
+
if (Object.keys(params).length > 0) {
|
|
18579
|
+
const paramsHash = await this.hashParams(params);
|
|
18580
|
+
keyParts.push(paramsHash);
|
|
18581
|
+
}
|
|
18582
|
+
return join(...keyParts) + ".json.gz";
|
|
18583
|
+
}
|
|
18584
|
+
async hashParams(params) {
|
|
18585
|
+
const sortedParams = Object.keys(params).sort().map((key) => `${key}:${params[key]}`).join("|") || "empty";
|
|
18586
|
+
return await sha256(sortedParams);
|
|
18587
|
+
}
|
|
18588
|
+
// Utility methods
|
|
18589
|
+
async getCacheStats() {
|
|
18590
|
+
if (!this.driver) return null;
|
|
18591
|
+
return {
|
|
18592
|
+
size: await this.driver.size(),
|
|
18593
|
+
keys: await this.driver.keys(),
|
|
18594
|
+
driver: this.driver.constructor.name
|
|
17814
18595
|
};
|
|
17815
|
-
|
|
17816
|
-
|
|
17817
|
-
|
|
17818
|
-
|
|
18596
|
+
}
|
|
18597
|
+
async clearAllCache() {
|
|
18598
|
+
if (!this.driver) return;
|
|
18599
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18600
|
+
if (resource.cache) {
|
|
18601
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
18602
|
+
await resource.cache.clear(keyPrefix);
|
|
18603
|
+
}
|
|
18604
|
+
}
|
|
18605
|
+
}
|
|
18606
|
+
async warmCache(resourceName, options = {}) {
|
|
18607
|
+
const resource = this.database.resources[resourceName];
|
|
18608
|
+
if (!resource) {
|
|
18609
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
18610
|
+
}
|
|
18611
|
+
const { includePartitions = true } = options;
|
|
18612
|
+
await resource.getAll();
|
|
18613
|
+
if (includePartitions && resource.config.partitions) {
|
|
18614
|
+
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
18615
|
+
if (partitionDef.fields) {
|
|
18616
|
+
const allRecords = await resource.getAll();
|
|
18617
|
+
const recordsArray = Array.isArray(allRecords) ? allRecords : [];
|
|
18618
|
+
const partitionValues = /* @__PURE__ */ new Set();
|
|
18619
|
+
for (const record of recordsArray.slice(0, 10)) {
|
|
18620
|
+
const values = this.getPartitionValues(record, resource);
|
|
18621
|
+
if (values[partitionName]) {
|
|
18622
|
+
partitionValues.add(JSON.stringify(values[partitionName]));
|
|
18623
|
+
}
|
|
18624
|
+
}
|
|
18625
|
+
for (const partitionValueStr of partitionValues) {
|
|
18626
|
+
const partitionValues2 = JSON.parse(partitionValueStr);
|
|
18627
|
+
await resource.list({ partition: partitionName, partitionValues: partitionValues2 });
|
|
18628
|
+
}
|
|
18629
|
+
}
|
|
18630
|
+
}
|
|
18631
|
+
}
|
|
18632
|
+
}
|
|
18633
|
+
}
|
|
18634
|
+
|
|
18635
|
+
const CostsPlugin = {
|
|
18636
|
+
async setup(db) {
|
|
18637
|
+
if (!db || !db.client) {
|
|
18638
|
+
return;
|
|
18639
|
+
}
|
|
18640
|
+
this.client = db.client;
|
|
18641
|
+
this.map = {
|
|
18642
|
+
PutObjectCommand: "put",
|
|
18643
|
+
GetObjectCommand: "get",
|
|
18644
|
+
HeadObjectCommand: "head",
|
|
18645
|
+
DeleteObjectCommand: "delete",
|
|
18646
|
+
DeleteObjectsCommand: "delete",
|
|
18647
|
+
ListObjectsV2Command: "list"
|
|
17819
18648
|
};
|
|
17820
|
-
|
|
17821
|
-
|
|
17822
|
-
|
|
17823
|
-
|
|
18649
|
+
this.costs = {
|
|
18650
|
+
total: 0,
|
|
18651
|
+
prices: {
|
|
18652
|
+
put: 5e-3 / 1e3,
|
|
18653
|
+
copy: 5e-3 / 1e3,
|
|
18654
|
+
list: 5e-3 / 1e3,
|
|
18655
|
+
post: 5e-3 / 1e3,
|
|
18656
|
+
get: 4e-4 / 1e3,
|
|
18657
|
+
select: 4e-4 / 1e3,
|
|
18658
|
+
delete: 4e-4 / 1e3,
|
|
18659
|
+
head: 4e-4 / 1e3
|
|
18660
|
+
},
|
|
18661
|
+
requests: {
|
|
18662
|
+
total: 0,
|
|
18663
|
+
put: 0,
|
|
18664
|
+
post: 0,
|
|
18665
|
+
copy: 0,
|
|
18666
|
+
list: 0,
|
|
18667
|
+
get: 0,
|
|
18668
|
+
select: 0,
|
|
18669
|
+
delete: 0,
|
|
18670
|
+
head: 0
|
|
18671
|
+
},
|
|
18672
|
+
events: {
|
|
18673
|
+
total: 0,
|
|
18674
|
+
PutObjectCommand: 0,
|
|
18675
|
+
GetObjectCommand: 0,
|
|
18676
|
+
HeadObjectCommand: 0,
|
|
18677
|
+
DeleteObjectCommand: 0,
|
|
18678
|
+
DeleteObjectsCommand: 0,
|
|
18679
|
+
ListObjectsV2Command: 0
|
|
18680
|
+
}
|
|
18681
|
+
};
|
|
18682
|
+
this.client.costs = JSON.parse(JSON.stringify(this.costs));
|
|
18683
|
+
},
|
|
18684
|
+
async start() {
|
|
18685
|
+
if (this.client) {
|
|
18686
|
+
this.client.on("command.response", (name) => this.addRequest(name, this.map[name]));
|
|
18687
|
+
this.client.on("command.error", (name) => this.addRequest(name, this.map[name]));
|
|
18688
|
+
}
|
|
18689
|
+
},
|
|
18690
|
+
addRequest(name, method) {
|
|
18691
|
+
if (!method) return;
|
|
18692
|
+
this.costs.events[name]++;
|
|
18693
|
+
this.costs.events.total++;
|
|
18694
|
+
this.costs.requests.total++;
|
|
18695
|
+
this.costs.requests[method]++;
|
|
18696
|
+
this.costs.total += this.costs.prices[method];
|
|
18697
|
+
if (this.client && this.client.costs) {
|
|
18698
|
+
this.client.costs.events[name]++;
|
|
18699
|
+
this.client.costs.events.total++;
|
|
18700
|
+
this.client.costs.requests.total++;
|
|
18701
|
+
this.client.costs.requests[method]++;
|
|
18702
|
+
this.client.costs.total += this.client.costs.prices[method];
|
|
18703
|
+
}
|
|
18704
|
+
}
|
|
18705
|
+
};
|
|
18706
|
+
|
|
18707
|
+
class FullTextPlugin extends Plugin {
|
|
18708
|
+
constructor(options = {}) {
|
|
18709
|
+
super();
|
|
18710
|
+
this.indexResource = null;
|
|
18711
|
+
this.config = {
|
|
18712
|
+
enabled: options.enabled !== false,
|
|
18713
|
+
minWordLength: options.minWordLength || 3,
|
|
18714
|
+
maxResults: options.maxResults || 100,
|
|
18715
|
+
...options
|
|
17824
18716
|
};
|
|
18717
|
+
this.indexes = /* @__PURE__ */ new Map();
|
|
18718
|
+
}
|
|
18719
|
+
async setup(database) {
|
|
18720
|
+
this.database = database;
|
|
18721
|
+
if (!this.config.enabled) return;
|
|
18722
|
+
try {
|
|
18723
|
+
this.indexResource = await database.createResource({
|
|
18724
|
+
name: "fulltext_indexes",
|
|
18725
|
+
attributes: {
|
|
18726
|
+
id: "string|required",
|
|
18727
|
+
resourceName: "string|required",
|
|
18728
|
+
fieldName: "string|required",
|
|
18729
|
+
word: "string|required",
|
|
18730
|
+
recordIds: "json|required",
|
|
18731
|
+
// Array of record IDs containing this word
|
|
18732
|
+
count: "number|required",
|
|
18733
|
+
lastUpdated: "string|required"
|
|
18734
|
+
}
|
|
18735
|
+
});
|
|
18736
|
+
} catch (error) {
|
|
18737
|
+
this.indexResource = database.resources.fulltext_indexes;
|
|
18738
|
+
}
|
|
18739
|
+
await this.loadIndexes();
|
|
18740
|
+
this.installIndexingHooks();
|
|
18741
|
+
}
|
|
18742
|
+
async start() {
|
|
18743
|
+
}
|
|
18744
|
+
async stop() {
|
|
18745
|
+
await this.saveIndexes();
|
|
18746
|
+
}
|
|
18747
|
+
async loadIndexes() {
|
|
18748
|
+
if (!this.indexResource) return;
|
|
18749
|
+
try {
|
|
18750
|
+
const allIndexes = await this.indexResource.getAll();
|
|
18751
|
+
for (const indexRecord of allIndexes) {
|
|
18752
|
+
const key = `${indexRecord.resourceName}:${indexRecord.fieldName}:${indexRecord.word}`;
|
|
18753
|
+
this.indexes.set(key, {
|
|
18754
|
+
recordIds: indexRecord.recordIds || [],
|
|
18755
|
+
count: indexRecord.count || 0
|
|
18756
|
+
});
|
|
18757
|
+
}
|
|
18758
|
+
} catch (error) {
|
|
18759
|
+
console.warn("Failed to load existing indexes:", error.message);
|
|
18760
|
+
}
|
|
18761
|
+
}
|
|
18762
|
+
async saveIndexes() {
|
|
18763
|
+
if (!this.indexResource) return;
|
|
18764
|
+
try {
|
|
18765
|
+
const existingIndexes = await this.indexResource.getAll();
|
|
18766
|
+
for (const index of existingIndexes) {
|
|
18767
|
+
await this.indexResource.delete(index.id);
|
|
18768
|
+
}
|
|
18769
|
+
for (const [key, data] of this.indexes.entries()) {
|
|
18770
|
+
const [resourceName, fieldName, word] = key.split(":");
|
|
18771
|
+
await this.indexResource.insert({
|
|
18772
|
+
id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
18773
|
+
resourceName,
|
|
18774
|
+
fieldName,
|
|
18775
|
+
word,
|
|
18776
|
+
recordIds: data.recordIds,
|
|
18777
|
+
count: data.count,
|
|
18778
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
18779
|
+
});
|
|
18780
|
+
}
|
|
18781
|
+
} catch (error) {
|
|
18782
|
+
console.error("Failed to save indexes:", error);
|
|
18783
|
+
}
|
|
18784
|
+
}
|
|
18785
|
+
installIndexingHooks() {
|
|
18786
|
+
if (!this.database.plugins) {
|
|
18787
|
+
this.database.plugins = {};
|
|
18788
|
+
}
|
|
18789
|
+
this.database.plugins.fulltext = this;
|
|
18790
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18791
|
+
if (resource.name === "fulltext_indexes") continue;
|
|
18792
|
+
this.installResourceHooks(resource);
|
|
18793
|
+
}
|
|
18794
|
+
if (!this.database._fulltextProxyInstalled) {
|
|
18795
|
+
this.database._previousCreateResourceForFullText = this.database.createResource;
|
|
18796
|
+
this.database.createResource = async function(...args) {
|
|
18797
|
+
const resource = await this._previousCreateResourceForFullText(...args);
|
|
18798
|
+
if (this.plugins?.fulltext && resource.name !== "fulltext_indexes") {
|
|
18799
|
+
this.plugins.fulltext.installResourceHooks(resource);
|
|
18800
|
+
}
|
|
18801
|
+
return resource;
|
|
18802
|
+
};
|
|
18803
|
+
this.database._fulltextProxyInstalled = true;
|
|
18804
|
+
}
|
|
18805
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
18806
|
+
if (resource.name !== "fulltext_indexes") {
|
|
18807
|
+
this.installResourceHooks(resource);
|
|
18808
|
+
}
|
|
18809
|
+
}
|
|
18810
|
+
}
|
|
18811
|
+
installResourceHooks(resource) {
|
|
18812
|
+
resource._insert = resource.insert;
|
|
18813
|
+
resource._update = resource.update;
|
|
18814
|
+
resource._delete = resource.delete;
|
|
18815
|
+
resource._deleteMany = resource.deleteMany;
|
|
18816
|
+
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
18817
|
+
const [data] = args;
|
|
18818
|
+
this.indexRecord(resource.name, result.id, data).catch(console.error);
|
|
18819
|
+
return result;
|
|
18820
|
+
});
|
|
18821
|
+
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
18822
|
+
const [id, data] = args;
|
|
18823
|
+
this.removeRecordFromIndex(resource.name, id).catch(console.error);
|
|
18824
|
+
this.indexRecord(resource.name, id, result).catch(console.error);
|
|
18825
|
+
return result;
|
|
18826
|
+
});
|
|
18827
|
+
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
18828
|
+
const [id] = args;
|
|
18829
|
+
this.removeRecordFromIndex(resource.name, id).catch(console.error);
|
|
18830
|
+
return result;
|
|
18831
|
+
});
|
|
18832
|
+
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
18833
|
+
const [ids] = args;
|
|
18834
|
+
for (const id of ids) {
|
|
18835
|
+
this.removeRecordFromIndex(resource.name, id).catch(console.error);
|
|
18836
|
+
}
|
|
18837
|
+
return result;
|
|
18838
|
+
});
|
|
18839
|
+
}
|
|
18840
|
+
async indexRecord(resourceName, recordId, data) {
|
|
18841
|
+
const indexedFields = this.getIndexedFields(resourceName);
|
|
18842
|
+
if (!indexedFields || indexedFields.length === 0) return;
|
|
18843
|
+
for (const fieldName of indexedFields) {
|
|
18844
|
+
const fieldValue = this.getFieldValue(data, fieldName);
|
|
18845
|
+
if (!fieldValue) continue;
|
|
18846
|
+
const words = this.tokenize(fieldValue);
|
|
18847
|
+
for (const word of words) {
|
|
18848
|
+
if (word.length < this.config.minWordLength) continue;
|
|
18849
|
+
const key = `${resourceName}:${fieldName}:${word.toLowerCase()}`;
|
|
18850
|
+
const existing = this.indexes.get(key) || { recordIds: [], count: 0 };
|
|
18851
|
+
if (!existing.recordIds.includes(recordId)) {
|
|
18852
|
+
existing.recordIds.push(recordId);
|
|
18853
|
+
existing.count = existing.recordIds.length;
|
|
18854
|
+
}
|
|
18855
|
+
this.indexes.set(key, existing);
|
|
18856
|
+
}
|
|
18857
|
+
}
|
|
18858
|
+
}
|
|
18859
|
+
async removeRecordFromIndex(resourceName, recordId) {
|
|
18860
|
+
for (const [key, data] of this.indexes.entries()) {
|
|
18861
|
+
if (key.startsWith(`${resourceName}:`)) {
|
|
18862
|
+
const index = data.recordIds.indexOf(recordId);
|
|
18863
|
+
if (index > -1) {
|
|
18864
|
+
data.recordIds.splice(index, 1);
|
|
18865
|
+
data.count = data.recordIds.length;
|
|
18866
|
+
if (data.recordIds.length === 0) {
|
|
18867
|
+
this.indexes.delete(key);
|
|
18868
|
+
} else {
|
|
18869
|
+
this.indexes.set(key, data);
|
|
18870
|
+
}
|
|
18871
|
+
}
|
|
18872
|
+
}
|
|
18873
|
+
}
|
|
18874
|
+
}
|
|
18875
|
+
getFieldValue(data, fieldPath) {
|
|
18876
|
+
if (!fieldPath.includes(".")) {
|
|
18877
|
+
return data[fieldPath];
|
|
18878
|
+
}
|
|
18879
|
+
const keys = fieldPath.split(".");
|
|
18880
|
+
let value = data;
|
|
18881
|
+
for (const key of keys) {
|
|
18882
|
+
if (value && typeof value === "object" && key in value) {
|
|
18883
|
+
value = value[key];
|
|
18884
|
+
} else {
|
|
18885
|
+
return null;
|
|
18886
|
+
}
|
|
18887
|
+
}
|
|
18888
|
+
return value;
|
|
18889
|
+
}
|
|
18890
|
+
tokenize(text) {
|
|
18891
|
+
if (!text) return [];
|
|
18892
|
+
const str = String(text).toLowerCase();
|
|
18893
|
+
return str.replace(/[^\w\s\u00C0-\u017F]/g, " ").split(/\s+/).filter((word) => word.length > 0);
|
|
18894
|
+
}
|
|
18895
|
+
getIndexedFields(resourceName) {
|
|
18896
|
+
if (this.config.fields) {
|
|
18897
|
+
return this.config.fields;
|
|
18898
|
+
}
|
|
18899
|
+
const fieldMappings = {
|
|
18900
|
+
users: ["name", "email"],
|
|
18901
|
+
products: ["name", "description"],
|
|
18902
|
+
articles: ["title", "content"]
|
|
18903
|
+
// Add more mappings as needed
|
|
18904
|
+
};
|
|
18905
|
+
return fieldMappings[resourceName] || [];
|
|
18906
|
+
}
|
|
18907
|
+
// Main search method
|
|
18908
|
+
async search(resourceName, query, options = {}) {
|
|
18909
|
+
const {
|
|
18910
|
+
fields = null,
|
|
18911
|
+
// Specific fields to search in
|
|
18912
|
+
limit = this.config.maxResults,
|
|
18913
|
+
offset = 0,
|
|
18914
|
+
exactMatch = false
|
|
18915
|
+
} = options;
|
|
18916
|
+
if (!query || query.trim().length === 0) {
|
|
18917
|
+
return [];
|
|
18918
|
+
}
|
|
18919
|
+
const searchWords = this.tokenize(query);
|
|
18920
|
+
const results = /* @__PURE__ */ new Map();
|
|
18921
|
+
const searchFields = fields || this.getIndexedFields(resourceName);
|
|
18922
|
+
if (searchFields.length === 0) {
|
|
18923
|
+
return [];
|
|
18924
|
+
}
|
|
18925
|
+
for (const word of searchWords) {
|
|
18926
|
+
if (word.length < this.config.minWordLength) continue;
|
|
18927
|
+
for (const fieldName of searchFields) {
|
|
18928
|
+
if (exactMatch) {
|
|
18929
|
+
const key = `${resourceName}:${fieldName}:${word.toLowerCase()}`;
|
|
18930
|
+
const indexData = this.indexes.get(key);
|
|
18931
|
+
if (indexData) {
|
|
18932
|
+
for (const recordId of indexData.recordIds) {
|
|
18933
|
+
const currentScore = results.get(recordId) || 0;
|
|
18934
|
+
results.set(recordId, currentScore + 1);
|
|
18935
|
+
}
|
|
18936
|
+
}
|
|
18937
|
+
} else {
|
|
18938
|
+
for (const [key, indexData] of this.indexes.entries()) {
|
|
18939
|
+
if (key.startsWith(`${resourceName}:${fieldName}:${word.toLowerCase()}`)) {
|
|
18940
|
+
for (const recordId of indexData.recordIds) {
|
|
18941
|
+
const currentScore = results.get(recordId) || 0;
|
|
18942
|
+
results.set(recordId, currentScore + 1);
|
|
18943
|
+
}
|
|
18944
|
+
}
|
|
18945
|
+
}
|
|
18946
|
+
}
|
|
18947
|
+
}
|
|
18948
|
+
}
|
|
18949
|
+
const sortedResults = Array.from(results.entries()).map(([recordId, score]) => ({ recordId, score })).sort((a, b) => b.score - a.score).slice(offset, offset + limit);
|
|
18950
|
+
return sortedResults;
|
|
18951
|
+
}
|
|
18952
|
+
// Search and return full records
|
|
18953
|
+
async searchRecords(resourceName, query, options = {}) {
|
|
18954
|
+
const searchResults = await this.search(resourceName, query, options);
|
|
18955
|
+
if (searchResults.length === 0) {
|
|
18956
|
+
return [];
|
|
18957
|
+
}
|
|
18958
|
+
const resource = this.database.resources[resourceName];
|
|
18959
|
+
if (!resource) {
|
|
18960
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
18961
|
+
}
|
|
18962
|
+
const recordIds = searchResults.map((result) => result.recordId);
|
|
18963
|
+
const records = await resource.getMany(recordIds);
|
|
18964
|
+
return records.map((record) => {
|
|
18965
|
+
const searchResult = searchResults.find((sr) => sr.recordId === record.id);
|
|
18966
|
+
return {
|
|
18967
|
+
...record,
|
|
18968
|
+
_searchScore: searchResult ? searchResult.score : 0
|
|
18969
|
+
};
|
|
18970
|
+
}).sort((a, b) => b._searchScore - a._searchScore);
|
|
18971
|
+
}
|
|
18972
|
+
// Utility methods
|
|
18973
|
+
async rebuildIndex(resourceName) {
|
|
18974
|
+
const resource = this.database.resources[resourceName];
|
|
18975
|
+
if (!resource) {
|
|
18976
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
18977
|
+
}
|
|
18978
|
+
for (const [key] of this.indexes.entries()) {
|
|
18979
|
+
if (key.startsWith(`${resourceName}:`)) {
|
|
18980
|
+
this.indexes.delete(key);
|
|
18981
|
+
}
|
|
18982
|
+
}
|
|
18983
|
+
const allRecords = await resource.getAll();
|
|
18984
|
+
const batchSize = 100;
|
|
18985
|
+
for (let i = 0; i < allRecords.length; i += batchSize) {
|
|
18986
|
+
const batch = allRecords.slice(i, i + batchSize);
|
|
18987
|
+
for (const record of batch) {
|
|
18988
|
+
await this.indexRecord(resourceName, record.id, record);
|
|
18989
|
+
}
|
|
18990
|
+
}
|
|
18991
|
+
await this.saveIndexes();
|
|
18992
|
+
}
|
|
18993
|
+
async getIndexStats() {
|
|
18994
|
+
const stats = {
|
|
18995
|
+
totalIndexes: this.indexes.size,
|
|
18996
|
+
resources: {},
|
|
18997
|
+
totalWords: 0
|
|
18998
|
+
};
|
|
18999
|
+
for (const [key, data] of this.indexes.entries()) {
|
|
19000
|
+
const [resourceName, fieldName] = key.split(":");
|
|
19001
|
+
if (!stats.resources[resourceName]) {
|
|
19002
|
+
stats.resources[resourceName] = {
|
|
19003
|
+
fields: {},
|
|
19004
|
+
totalRecords: /* @__PURE__ */ new Set(),
|
|
19005
|
+
totalWords: 0
|
|
19006
|
+
};
|
|
19007
|
+
}
|
|
19008
|
+
if (!stats.resources[resourceName].fields[fieldName]) {
|
|
19009
|
+
stats.resources[resourceName].fields[fieldName] = {
|
|
19010
|
+
words: 0,
|
|
19011
|
+
totalOccurrences: 0
|
|
19012
|
+
};
|
|
19013
|
+
}
|
|
19014
|
+
stats.resources[resourceName].fields[fieldName].words++;
|
|
19015
|
+
stats.resources[resourceName].fields[fieldName].totalOccurrences += data.count;
|
|
19016
|
+
stats.resources[resourceName].totalWords++;
|
|
19017
|
+
for (const recordId of data.recordIds) {
|
|
19018
|
+
stats.resources[resourceName].totalRecords.add(recordId);
|
|
19019
|
+
}
|
|
19020
|
+
stats.totalWords++;
|
|
19021
|
+
}
|
|
19022
|
+
for (const resourceName in stats.resources) {
|
|
19023
|
+
stats.resources[resourceName].totalRecords = stats.resources[resourceName].totalRecords.size;
|
|
19024
|
+
}
|
|
19025
|
+
return stats;
|
|
19026
|
+
}
|
|
19027
|
+
async rebuildAllIndexes({ timeout } = {}) {
|
|
19028
|
+
if (timeout) {
|
|
19029
|
+
return Promise.race([
|
|
19030
|
+
this._rebuildAllIndexesInternal(),
|
|
19031
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error("Timeout")), timeout))
|
|
19032
|
+
]);
|
|
19033
|
+
}
|
|
19034
|
+
return this._rebuildAllIndexesInternal();
|
|
19035
|
+
}
|
|
19036
|
+
async _rebuildAllIndexesInternal() {
|
|
19037
|
+
const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "fulltext_indexes");
|
|
19038
|
+
for (const resourceName of resourceNames) {
|
|
19039
|
+
try {
|
|
19040
|
+
await this.rebuildIndex(resourceName);
|
|
19041
|
+
} catch (error) {
|
|
19042
|
+
console.warn(`Failed to rebuild index for resource ${resourceName}:`, error.message);
|
|
19043
|
+
}
|
|
19044
|
+
}
|
|
19045
|
+
}
|
|
19046
|
+
async clearIndex(resourceName) {
|
|
19047
|
+
for (const [key] of this.indexes.entries()) {
|
|
19048
|
+
if (key.startsWith(`${resourceName}:`)) {
|
|
19049
|
+
this.indexes.delete(key);
|
|
19050
|
+
}
|
|
19051
|
+
}
|
|
19052
|
+
await this.saveIndexes();
|
|
19053
|
+
}
|
|
19054
|
+
async clearAllIndexes() {
|
|
19055
|
+
this.indexes.clear();
|
|
19056
|
+
await this.saveIndexes();
|
|
19057
|
+
}
|
|
19058
|
+
}
|
|
19059
|
+
|
|
19060
|
+
class MetricsPlugin extends Plugin {
|
|
19061
|
+
constructor(options = {}) {
|
|
19062
|
+
super();
|
|
19063
|
+
this.config = {
|
|
19064
|
+
enabled: options.enabled !== false,
|
|
19065
|
+
collectPerformance: options.collectPerformance !== false,
|
|
19066
|
+
collectErrors: options.collectErrors !== false,
|
|
19067
|
+
collectUsage: options.collectUsage !== false,
|
|
19068
|
+
retentionDays: options.retentionDays || 30,
|
|
19069
|
+
flushInterval: options.flushInterval || 6e4,
|
|
19070
|
+
// 1 minute
|
|
19071
|
+
...options
|
|
19072
|
+
};
|
|
19073
|
+
this.metrics = {
|
|
19074
|
+
operations: {
|
|
19075
|
+
insert: { count: 0, totalTime: 0, errors: 0 },
|
|
19076
|
+
update: { count: 0, totalTime: 0, errors: 0 },
|
|
19077
|
+
delete: { count: 0, totalTime: 0, errors: 0 },
|
|
19078
|
+
get: { count: 0, totalTime: 0, errors: 0 },
|
|
19079
|
+
list: { count: 0, totalTime: 0, errors: 0 },
|
|
19080
|
+
count: { count: 0, totalTime: 0, errors: 0 }
|
|
19081
|
+
},
|
|
19082
|
+
resources: {},
|
|
19083
|
+
errors: [],
|
|
19084
|
+
performance: [],
|
|
19085
|
+
startTime: (/* @__PURE__ */ new Date()).toISOString()
|
|
19086
|
+
};
|
|
19087
|
+
this.flushTimer = null;
|
|
19088
|
+
}
|
|
19089
|
+
async setup(database) {
|
|
19090
|
+
this.database = database;
|
|
19091
|
+
if (!this.config.enabled || process.env.NODE_ENV === "test") return;
|
|
19092
|
+
try {
|
|
19093
|
+
this.metricsResource = await database.createResource({
|
|
19094
|
+
name: "metrics",
|
|
19095
|
+
attributes: {
|
|
19096
|
+
id: "string|required",
|
|
19097
|
+
type: "string|required",
|
|
19098
|
+
// 'operation', 'error', 'performance'
|
|
19099
|
+
resourceName: "string",
|
|
19100
|
+
operation: "string",
|
|
19101
|
+
count: "number|required",
|
|
19102
|
+
totalTime: "number|required",
|
|
19103
|
+
errors: "number|required",
|
|
19104
|
+
avgTime: "number|required",
|
|
19105
|
+
timestamp: "string|required",
|
|
19106
|
+
metadata: "json"
|
|
19107
|
+
}
|
|
19108
|
+
});
|
|
19109
|
+
this.errorsResource = await database.createResource({
|
|
19110
|
+
name: "error_logs",
|
|
19111
|
+
attributes: {
|
|
19112
|
+
id: "string|required",
|
|
19113
|
+
resourceName: "string|required",
|
|
19114
|
+
operation: "string|required",
|
|
19115
|
+
error: "string|required",
|
|
19116
|
+
timestamp: "string|required",
|
|
19117
|
+
metadata: "json"
|
|
19118
|
+
}
|
|
19119
|
+
});
|
|
19120
|
+
this.performanceResource = await database.createResource({
|
|
19121
|
+
name: "performance_logs",
|
|
19122
|
+
attributes: {
|
|
19123
|
+
id: "string|required",
|
|
19124
|
+
resourceName: "string|required",
|
|
19125
|
+
operation: "string|required",
|
|
19126
|
+
duration: "number|required",
|
|
19127
|
+
timestamp: "string|required",
|
|
19128
|
+
metadata: "json"
|
|
19129
|
+
}
|
|
19130
|
+
});
|
|
19131
|
+
} catch (error) {
|
|
19132
|
+
this.metricsResource = database.resources.metrics;
|
|
19133
|
+
this.errorsResource = database.resources.error_logs;
|
|
19134
|
+
this.performanceResource = database.resources.performance_logs;
|
|
19135
|
+
}
|
|
19136
|
+
this.installMetricsHooks();
|
|
19137
|
+
if (process.env.NODE_ENV !== "test") {
|
|
19138
|
+
this.startFlushTimer();
|
|
19139
|
+
}
|
|
19140
|
+
}
|
|
19141
|
+
async start() {
|
|
19142
|
+
}
|
|
19143
|
+
async stop() {
|
|
19144
|
+
if (this.flushTimer) {
|
|
19145
|
+
clearInterval(this.flushTimer);
|
|
19146
|
+
this.flushTimer = null;
|
|
19147
|
+
}
|
|
19148
|
+
if (process.env.NODE_ENV !== "test") {
|
|
19149
|
+
await this.flushMetrics();
|
|
19150
|
+
}
|
|
19151
|
+
}
|
|
19152
|
+
installMetricsHooks() {
|
|
19153
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
19154
|
+
if (["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
|
|
19155
|
+
continue;
|
|
19156
|
+
}
|
|
19157
|
+
this.installResourceHooks(resource);
|
|
19158
|
+
}
|
|
19159
|
+
this.database._createResource = this.database.createResource;
|
|
19160
|
+
this.database.createResource = async function(...args) {
|
|
19161
|
+
const resource = await this._createResource(...args);
|
|
19162
|
+
if (this.plugins?.metrics && !["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
|
|
19163
|
+
this.plugins.metrics.installResourceHooks(resource);
|
|
19164
|
+
}
|
|
19165
|
+
return resource;
|
|
19166
|
+
};
|
|
19167
|
+
}
|
|
19168
|
+
installResourceHooks(resource) {
|
|
19169
|
+
resource._insert = resource.insert;
|
|
19170
|
+
resource._update = resource.update;
|
|
19171
|
+
resource._delete = resource.delete;
|
|
19172
|
+
resource._deleteMany = resource.deleteMany;
|
|
19173
|
+
resource._get = resource.get;
|
|
19174
|
+
resource._getMany = resource.getMany;
|
|
19175
|
+
resource._getAll = resource.getAll;
|
|
19176
|
+
resource._list = resource.list;
|
|
19177
|
+
resource._listIds = resource.listIds;
|
|
19178
|
+
resource._count = resource.count;
|
|
19179
|
+
resource._page = resource.page;
|
|
19180
|
+
resource.insert = async function(...args) {
|
|
19181
|
+
const startTime = Date.now();
|
|
19182
|
+
try {
|
|
19183
|
+
const result = await resource._insert(...args);
|
|
19184
|
+
this.recordOperation(resource.name, "insert", Date.now() - startTime, false);
|
|
19185
|
+
return result;
|
|
19186
|
+
} catch (error) {
|
|
19187
|
+
this.recordOperation(resource.name, "insert", Date.now() - startTime, true);
|
|
19188
|
+
this.recordError(resource.name, "insert", error);
|
|
19189
|
+
throw error;
|
|
19190
|
+
}
|
|
19191
|
+
}.bind(this);
|
|
19192
|
+
resource.update = async function(...args) {
|
|
19193
|
+
const startTime = Date.now();
|
|
19194
|
+
try {
|
|
19195
|
+
const result = await resource._update(...args);
|
|
19196
|
+
this.recordOperation(resource.name, "update", Date.now() - startTime, false);
|
|
19197
|
+
return result;
|
|
19198
|
+
} catch (error) {
|
|
19199
|
+
this.recordOperation(resource.name, "update", Date.now() - startTime, true);
|
|
19200
|
+
this.recordError(resource.name, "update", error);
|
|
19201
|
+
throw error;
|
|
19202
|
+
}
|
|
19203
|
+
}.bind(this);
|
|
19204
|
+
resource.delete = async function(...args) {
|
|
19205
|
+
const startTime = Date.now();
|
|
19206
|
+
try {
|
|
19207
|
+
const result = await resource._delete(...args);
|
|
19208
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, false);
|
|
19209
|
+
return result;
|
|
19210
|
+
} catch (error) {
|
|
19211
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, true);
|
|
19212
|
+
this.recordError(resource.name, "delete", error);
|
|
19213
|
+
throw error;
|
|
19214
|
+
}
|
|
19215
|
+
}.bind(this);
|
|
19216
|
+
resource.deleteMany = async function(...args) {
|
|
19217
|
+
const startTime = Date.now();
|
|
19218
|
+
try {
|
|
19219
|
+
const result = await resource._deleteMany(...args);
|
|
19220
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, false);
|
|
19221
|
+
return result;
|
|
19222
|
+
} catch (error) {
|
|
19223
|
+
this.recordOperation(resource.name, "delete", Date.now() - startTime, true);
|
|
19224
|
+
this.recordError(resource.name, "delete", error);
|
|
19225
|
+
throw error;
|
|
19226
|
+
}
|
|
19227
|
+
}.bind(this);
|
|
19228
|
+
resource.get = async function(...args) {
|
|
19229
|
+
const startTime = Date.now();
|
|
19230
|
+
try {
|
|
19231
|
+
const result = await resource._get(...args);
|
|
19232
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, false);
|
|
19233
|
+
return result;
|
|
19234
|
+
} catch (error) {
|
|
19235
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, true);
|
|
19236
|
+
this.recordError(resource.name, "get", error);
|
|
19237
|
+
throw error;
|
|
19238
|
+
}
|
|
19239
|
+
}.bind(this);
|
|
19240
|
+
resource.getMany = async function(...args) {
|
|
19241
|
+
const startTime = Date.now();
|
|
19242
|
+
try {
|
|
19243
|
+
const result = await resource._getMany(...args);
|
|
19244
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, false);
|
|
19245
|
+
return result;
|
|
19246
|
+
} catch (error) {
|
|
19247
|
+
this.recordOperation(resource.name, "get", Date.now() - startTime, true);
|
|
19248
|
+
this.recordError(resource.name, "get", error);
|
|
19249
|
+
throw error;
|
|
19250
|
+
}
|
|
19251
|
+
}.bind(this);
|
|
19252
|
+
resource.getAll = async function(...args) {
|
|
19253
|
+
const startTime = Date.now();
|
|
19254
|
+
try {
|
|
19255
|
+
const result = await resource._getAll(...args);
|
|
19256
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19257
|
+
return result;
|
|
19258
|
+
} catch (error) {
|
|
19259
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19260
|
+
this.recordError(resource.name, "list", error);
|
|
19261
|
+
throw error;
|
|
19262
|
+
}
|
|
19263
|
+
}.bind(this);
|
|
19264
|
+
resource.list = async function(...args) {
|
|
19265
|
+
const startTime = Date.now();
|
|
19266
|
+
try {
|
|
19267
|
+
const result = await resource._list(...args);
|
|
19268
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19269
|
+
return result;
|
|
19270
|
+
} catch (error) {
|
|
19271
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19272
|
+
this.recordError(resource.name, "list", error);
|
|
19273
|
+
throw error;
|
|
19274
|
+
}
|
|
19275
|
+
}.bind(this);
|
|
19276
|
+
resource.listIds = async function(...args) {
|
|
19277
|
+
const startTime = Date.now();
|
|
19278
|
+
try {
|
|
19279
|
+
const result = await resource._listIds(...args);
|
|
19280
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19281
|
+
return result;
|
|
19282
|
+
} catch (error) {
|
|
19283
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19284
|
+
this.recordError(resource.name, "list", error);
|
|
19285
|
+
throw error;
|
|
19286
|
+
}
|
|
19287
|
+
}.bind(this);
|
|
19288
|
+
resource.count = async function(...args) {
|
|
19289
|
+
const startTime = Date.now();
|
|
19290
|
+
try {
|
|
19291
|
+
const result = await resource._count(...args);
|
|
19292
|
+
this.recordOperation(resource.name, "count", Date.now() - startTime, false);
|
|
19293
|
+
return result;
|
|
19294
|
+
} catch (error) {
|
|
19295
|
+
this.recordOperation(resource.name, "count", Date.now() - startTime, true);
|
|
19296
|
+
this.recordError(resource.name, "count", error);
|
|
19297
|
+
throw error;
|
|
19298
|
+
}
|
|
19299
|
+
}.bind(this);
|
|
19300
|
+
resource.page = async function(...args) {
|
|
19301
|
+
const startTime = Date.now();
|
|
19302
|
+
try {
|
|
19303
|
+
const result = await resource._page(...args);
|
|
19304
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, false);
|
|
19305
|
+
return result;
|
|
19306
|
+
} catch (error) {
|
|
19307
|
+
this.recordOperation(resource.name, "list", Date.now() - startTime, true);
|
|
19308
|
+
this.recordError(resource.name, "list", error);
|
|
19309
|
+
throw error;
|
|
19310
|
+
}
|
|
19311
|
+
}.bind(this);
|
|
19312
|
+
}
|
|
19313
|
+
recordOperation(resourceName, operation, duration, isError) {
|
|
19314
|
+
if (this.metrics.operations[operation]) {
|
|
19315
|
+
this.metrics.operations[operation].count++;
|
|
19316
|
+
this.metrics.operations[operation].totalTime += duration;
|
|
19317
|
+
if (isError) {
|
|
19318
|
+
this.metrics.operations[operation].errors++;
|
|
19319
|
+
}
|
|
19320
|
+
}
|
|
19321
|
+
if (!this.metrics.resources[resourceName]) {
|
|
19322
|
+
this.metrics.resources[resourceName] = {
|
|
19323
|
+
insert: { count: 0, totalTime: 0, errors: 0 },
|
|
19324
|
+
update: { count: 0, totalTime: 0, errors: 0 },
|
|
19325
|
+
delete: { count: 0, totalTime: 0, errors: 0 },
|
|
19326
|
+
get: { count: 0, totalTime: 0, errors: 0 },
|
|
19327
|
+
list: { count: 0, totalTime: 0, errors: 0 },
|
|
19328
|
+
count: { count: 0, totalTime: 0, errors: 0 }
|
|
19329
|
+
};
|
|
19330
|
+
}
|
|
19331
|
+
if (this.metrics.resources[resourceName][operation]) {
|
|
19332
|
+
this.metrics.resources[resourceName][operation].count++;
|
|
19333
|
+
this.metrics.resources[resourceName][operation].totalTime += duration;
|
|
19334
|
+
if (isError) {
|
|
19335
|
+
this.metrics.resources[resourceName][operation].errors++;
|
|
19336
|
+
}
|
|
19337
|
+
}
|
|
19338
|
+
if (this.config.collectPerformance) {
|
|
19339
|
+
this.metrics.performance.push({
|
|
19340
|
+
resourceName,
|
|
19341
|
+
operation,
|
|
19342
|
+
duration,
|
|
19343
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
19344
|
+
});
|
|
19345
|
+
}
|
|
19346
|
+
}
|
|
19347
|
+
recordError(resourceName, operation, error) {
|
|
19348
|
+
if (!this.config.collectErrors) return;
|
|
19349
|
+
this.metrics.errors.push({
|
|
19350
|
+
resourceName,
|
|
19351
|
+
operation,
|
|
19352
|
+
error: error.message,
|
|
19353
|
+
stack: error.stack,
|
|
19354
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
19355
|
+
});
|
|
19356
|
+
}
|
|
19357
|
+
startFlushTimer() {
|
|
19358
|
+
if (this.flushTimer) {
|
|
19359
|
+
clearInterval(this.flushTimer);
|
|
19360
|
+
}
|
|
19361
|
+
if (this.config.flushInterval > 0) {
|
|
19362
|
+
this.flushTimer = setInterval(() => {
|
|
19363
|
+
this.flushMetrics().catch(console.error);
|
|
19364
|
+
}, this.config.flushInterval);
|
|
19365
|
+
}
|
|
19366
|
+
}
|
|
19367
|
+
async flushMetrics() {
|
|
19368
|
+
if (!this.metricsResource) return;
|
|
19369
|
+
try {
|
|
19370
|
+
const metadata = process.env.NODE_ENV === "test" ? {} : { global: "true" };
|
|
19371
|
+
const perfMetadata = process.env.NODE_ENV === "test" ? {} : { perf: "true" };
|
|
19372
|
+
const errorMetadata = process.env.NODE_ENV === "test" ? {} : { error: "true" };
|
|
19373
|
+
const resourceMetadata = process.env.NODE_ENV === "test" ? {} : { resource: "true" };
|
|
19374
|
+
for (const [operation, data] of Object.entries(this.metrics.operations)) {
|
|
19375
|
+
if (data.count > 0) {
|
|
19376
|
+
await this.metricsResource.insert({
|
|
19377
|
+
id: `metrics-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19378
|
+
type: "operation",
|
|
19379
|
+
resourceName: "global",
|
|
19380
|
+
operation,
|
|
19381
|
+
count: data.count,
|
|
19382
|
+
totalTime: data.totalTime,
|
|
19383
|
+
errors: data.errors,
|
|
19384
|
+
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
19385
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19386
|
+
metadata
|
|
19387
|
+
});
|
|
19388
|
+
}
|
|
19389
|
+
}
|
|
19390
|
+
for (const [resourceName, operations] of Object.entries(this.metrics.resources)) {
|
|
19391
|
+
for (const [operation, data] of Object.entries(operations)) {
|
|
19392
|
+
if (data.count > 0) {
|
|
19393
|
+
await this.metricsResource.insert({
|
|
19394
|
+
id: `metrics-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19395
|
+
type: "operation",
|
|
19396
|
+
resourceName,
|
|
19397
|
+
operation,
|
|
19398
|
+
count: data.count,
|
|
19399
|
+
totalTime: data.totalTime,
|
|
19400
|
+
errors: data.errors,
|
|
19401
|
+
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
19402
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19403
|
+
metadata: resourceMetadata
|
|
19404
|
+
});
|
|
19405
|
+
}
|
|
19406
|
+
}
|
|
19407
|
+
}
|
|
19408
|
+
if (this.config.collectPerformance && this.metrics.performance.length > 0) {
|
|
19409
|
+
for (const perf of this.metrics.performance) {
|
|
19410
|
+
await this.performanceResource.insert({
|
|
19411
|
+
id: `perf-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19412
|
+
resourceName: perf.resourceName,
|
|
19413
|
+
operation: perf.operation,
|
|
19414
|
+
duration: perf.duration,
|
|
19415
|
+
timestamp: perf.timestamp,
|
|
19416
|
+
metadata: perfMetadata
|
|
19417
|
+
});
|
|
19418
|
+
}
|
|
19419
|
+
}
|
|
19420
|
+
if (this.config.collectErrors && this.metrics.errors.length > 0) {
|
|
19421
|
+
for (const error of this.metrics.errors) {
|
|
19422
|
+
await this.errorsResource.insert({
|
|
19423
|
+
id: `error-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
19424
|
+
resourceName: error.resourceName,
|
|
19425
|
+
operation: error.operation,
|
|
19426
|
+
error: error.error,
|
|
19427
|
+
stack: error.stack,
|
|
19428
|
+
timestamp: error.timestamp,
|
|
19429
|
+
metadata: errorMetadata
|
|
19430
|
+
});
|
|
19431
|
+
}
|
|
19432
|
+
}
|
|
19433
|
+
this.resetMetrics();
|
|
19434
|
+
} catch (error) {
|
|
19435
|
+
console.error("Failed to flush metrics:", error);
|
|
19436
|
+
}
|
|
19437
|
+
}
|
|
19438
|
+
resetMetrics() {
|
|
19439
|
+
for (const operation of Object.keys(this.metrics.operations)) {
|
|
19440
|
+
this.metrics.operations[operation] = { count: 0, totalTime: 0, errors: 0 };
|
|
19441
|
+
}
|
|
19442
|
+
for (const resourceName of Object.keys(this.metrics.resources)) {
|
|
19443
|
+
for (const operation of Object.keys(this.metrics.resources[resourceName])) {
|
|
19444
|
+
this.metrics.resources[resourceName][operation] = { count: 0, totalTime: 0, errors: 0 };
|
|
19445
|
+
}
|
|
19446
|
+
}
|
|
19447
|
+
this.metrics.performance = [];
|
|
19448
|
+
this.metrics.errors = [];
|
|
19449
|
+
}
|
|
19450
|
+
// Utility methods
|
|
19451
|
+
async getMetrics(options = {}) {
|
|
19452
|
+
const {
|
|
19453
|
+
type = "operation",
|
|
19454
|
+
resourceName,
|
|
19455
|
+
operation,
|
|
19456
|
+
startDate,
|
|
19457
|
+
endDate,
|
|
19458
|
+
limit = 100,
|
|
19459
|
+
offset = 0
|
|
19460
|
+
} = options;
|
|
19461
|
+
if (!this.metricsResource) return [];
|
|
19462
|
+
const allMetrics = await this.metricsResource.getAll();
|
|
19463
|
+
let filtered = allMetrics.filter((metric) => {
|
|
19464
|
+
if (type && metric.type !== type) return false;
|
|
19465
|
+
if (resourceName && metric.resourceName !== resourceName) return false;
|
|
19466
|
+
if (operation && metric.operation !== operation) return false;
|
|
19467
|
+
if (startDate && new Date(metric.timestamp) < new Date(startDate)) return false;
|
|
19468
|
+
if (endDate && new Date(metric.timestamp) > new Date(endDate)) return false;
|
|
19469
|
+
return true;
|
|
19470
|
+
});
|
|
19471
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
19472
|
+
return filtered.slice(offset, offset + limit);
|
|
19473
|
+
}
|
|
19474
|
+
async getErrorLogs(options = {}) {
|
|
19475
|
+
if (!this.errorsResource) return [];
|
|
19476
|
+
const {
|
|
19477
|
+
resourceName,
|
|
19478
|
+
operation,
|
|
19479
|
+
startDate,
|
|
19480
|
+
endDate,
|
|
19481
|
+
limit = 100,
|
|
19482
|
+
offset = 0
|
|
19483
|
+
} = options;
|
|
19484
|
+
const allErrors = await this.errorsResource.getAll();
|
|
19485
|
+
let filtered = allErrors.filter((error) => {
|
|
19486
|
+
if (resourceName && error.resourceName !== resourceName) return false;
|
|
19487
|
+
if (operation && error.operation !== operation) return false;
|
|
19488
|
+
if (startDate && new Date(error.timestamp) < new Date(startDate)) return false;
|
|
19489
|
+
if (endDate && new Date(error.timestamp) > new Date(endDate)) return false;
|
|
19490
|
+
return true;
|
|
19491
|
+
});
|
|
19492
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
19493
|
+
return filtered.slice(offset, offset + limit);
|
|
19494
|
+
}
|
|
19495
|
+
async getPerformanceLogs(options = {}) {
|
|
19496
|
+
if (!this.performanceResource) return [];
|
|
19497
|
+
const {
|
|
19498
|
+
resourceName,
|
|
19499
|
+
operation,
|
|
19500
|
+
startDate,
|
|
19501
|
+
endDate,
|
|
19502
|
+
limit = 100,
|
|
19503
|
+
offset = 0
|
|
19504
|
+
} = options;
|
|
19505
|
+
const allPerformance = await this.performanceResource.getAll();
|
|
19506
|
+
let filtered = allPerformance.filter((perf) => {
|
|
19507
|
+
if (resourceName && perf.resourceName !== resourceName) return false;
|
|
19508
|
+
if (operation && perf.operation !== operation) return false;
|
|
19509
|
+
if (startDate && new Date(perf.timestamp) < new Date(startDate)) return false;
|
|
19510
|
+
if (endDate && new Date(perf.timestamp) > new Date(endDate)) return false;
|
|
19511
|
+
return true;
|
|
19512
|
+
});
|
|
19513
|
+
filtered.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
|
|
19514
|
+
return filtered.slice(offset, offset + limit);
|
|
19515
|
+
}
|
|
19516
|
+
async getStats() {
|
|
19517
|
+
const now = /* @__PURE__ */ new Date();
|
|
19518
|
+
const startDate = new Date(now.getTime() - 24 * 60 * 60 * 1e3);
|
|
19519
|
+
const [metrics, errors, performance] = await Promise.all([
|
|
19520
|
+
this.getMetrics({ startDate: startDate.toISOString() }),
|
|
19521
|
+
this.getErrorLogs({ startDate: startDate.toISOString() }),
|
|
19522
|
+
this.getPerformanceLogs({ startDate: startDate.toISOString() })
|
|
19523
|
+
]);
|
|
19524
|
+
const stats = {
|
|
19525
|
+
period: "24h",
|
|
19526
|
+
totalOperations: 0,
|
|
19527
|
+
totalErrors: errors.length,
|
|
19528
|
+
avgResponseTime: 0,
|
|
19529
|
+
operationsByType: {},
|
|
19530
|
+
resources: {},
|
|
19531
|
+
uptime: {
|
|
19532
|
+
startTime: this.metrics.startTime,
|
|
19533
|
+
duration: now.getTime() - new Date(this.metrics.startTime).getTime()
|
|
19534
|
+
}
|
|
19535
|
+
};
|
|
19536
|
+
for (const metric of metrics) {
|
|
19537
|
+
if (metric.type === "operation") {
|
|
19538
|
+
stats.totalOperations += metric.count;
|
|
19539
|
+
if (!stats.operationsByType[metric.operation]) {
|
|
19540
|
+
stats.operationsByType[metric.operation] = {
|
|
19541
|
+
count: 0,
|
|
19542
|
+
errors: 0,
|
|
19543
|
+
avgTime: 0
|
|
19544
|
+
};
|
|
19545
|
+
}
|
|
19546
|
+
stats.operationsByType[metric.operation].count += metric.count;
|
|
19547
|
+
stats.operationsByType[metric.operation].errors += metric.errors;
|
|
19548
|
+
const current = stats.operationsByType[metric.operation];
|
|
19549
|
+
const totalCount2 = current.count;
|
|
19550
|
+
const newAvg = (current.avgTime * (totalCount2 - metric.count) + metric.totalTime) / totalCount2;
|
|
19551
|
+
current.avgTime = newAvg;
|
|
19552
|
+
}
|
|
19553
|
+
}
|
|
19554
|
+
const totalTime = metrics.reduce((sum, m) => sum + m.totalTime, 0);
|
|
19555
|
+
const totalCount = metrics.reduce((sum, m) => sum + m.count, 0);
|
|
19556
|
+
stats.avgResponseTime = totalCount > 0 ? totalTime / totalCount : 0;
|
|
19557
|
+
return stats;
|
|
19558
|
+
}
|
|
19559
|
+
async cleanupOldData() {
|
|
19560
|
+
const cutoffDate = /* @__PURE__ */ new Date();
|
|
19561
|
+
cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
|
|
19562
|
+
if (this.metricsResource) {
|
|
19563
|
+
const oldMetrics = await this.getMetrics({ endDate: cutoffDate.toISOString() });
|
|
19564
|
+
for (const metric of oldMetrics) {
|
|
19565
|
+
await this.metricsResource.delete(metric.id);
|
|
19566
|
+
}
|
|
19567
|
+
}
|
|
19568
|
+
if (this.errorsResource) {
|
|
19569
|
+
const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
|
|
19570
|
+
for (const error of oldErrors) {
|
|
19571
|
+
await this.errorsResource.delete(error.id);
|
|
19572
|
+
}
|
|
19573
|
+
}
|
|
19574
|
+
if (this.performanceResource) {
|
|
19575
|
+
const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
|
|
19576
|
+
for (const perf of oldPerformance) {
|
|
19577
|
+
await this.performanceResource.delete(perf.id);
|
|
19578
|
+
}
|
|
19579
|
+
}
|
|
19580
|
+
console.log(`Cleaned up data older than ${this.config.retentionDays} days`);
|
|
19581
|
+
}
|
|
19582
|
+
}
|
|
19583
|
+
|
|
19584
|
+
class BaseReplicator extends EventEmitter {
|
|
19585
|
+
constructor(config = {}) {
|
|
19586
|
+
super();
|
|
19587
|
+
this.config = config;
|
|
19588
|
+
this.name = this.constructor.name;
|
|
19589
|
+
this.enabled = config.enabled !== false;
|
|
19590
|
+
}
|
|
19591
|
+
/**
|
|
19592
|
+
* Initialize the replicator
|
|
19593
|
+
* @param {Object} database - The s3db database instance
|
|
19594
|
+
* @returns {Promise<void>}
|
|
19595
|
+
*/
|
|
19596
|
+
async initialize(database) {
|
|
19597
|
+
this.database = database;
|
|
19598
|
+
this.emit("initialized", { replicator: this.name });
|
|
19599
|
+
}
|
|
19600
|
+
/**
|
|
19601
|
+
* Replicate data to the target
|
|
19602
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
19603
|
+
* @param {string} operation - Operation type (insert, update, delete)
|
|
19604
|
+
* @param {Object} data - The data to replicate
|
|
19605
|
+
* @param {string} id - Record ID
|
|
19606
|
+
* @returns {Promise<Object>} Replication result
|
|
19607
|
+
*/
|
|
19608
|
+
async replicate(resourceName, operation, data, id) {
|
|
19609
|
+
throw new Error(`replicate() method must be implemented by ${this.name}`);
|
|
19610
|
+
}
|
|
19611
|
+
/**
|
|
19612
|
+
* Replicate multiple records in batch
|
|
19613
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
19614
|
+
* @param {Array} records - Array of records to replicate
|
|
19615
|
+
* @returns {Promise<Object>} Batch replication result
|
|
19616
|
+
*/
|
|
19617
|
+
async replicateBatch(resourceName, records) {
|
|
19618
|
+
throw new Error(`replicateBatch() method must be implemented by ${this.name}`);
|
|
19619
|
+
}
|
|
19620
|
+
/**
|
|
19621
|
+
* Test the connection to the target
|
|
19622
|
+
* @returns {Promise<boolean>} True if connection is successful
|
|
19623
|
+
*/
|
|
19624
|
+
async testConnection() {
|
|
19625
|
+
throw new Error(`testConnection() method must be implemented by ${this.name}`);
|
|
19626
|
+
}
|
|
19627
|
+
/**
|
|
19628
|
+
* Get replicator status and statistics
|
|
19629
|
+
* @returns {Promise<Object>} Status information
|
|
19630
|
+
*/
|
|
19631
|
+
async getStatus() {
|
|
19632
|
+
return {
|
|
19633
|
+
name: this.name,
|
|
19634
|
+
enabled: this.enabled,
|
|
19635
|
+
config: this.config,
|
|
19636
|
+
connected: false
|
|
19637
|
+
};
|
|
19638
|
+
}
|
|
19639
|
+
/**
|
|
19640
|
+
* Cleanup resources
|
|
19641
|
+
* @returns {Promise<void>}
|
|
19642
|
+
*/
|
|
19643
|
+
async cleanup() {
|
|
19644
|
+
this.emit("cleanup", { replicator: this.name });
|
|
19645
|
+
}
|
|
19646
|
+
/**
|
|
19647
|
+
* Validate replicator configuration
|
|
19648
|
+
* @returns {Object} Validation result
|
|
19649
|
+
*/
|
|
19650
|
+
validateConfig() {
|
|
19651
|
+
return { isValid: true, errors: [] };
|
|
19652
|
+
}
|
|
19653
|
+
}
|
|
19654
|
+
|
|
19655
|
+
class S3dbReplicator extends BaseReplicator {
|
|
19656
|
+
constructor(config = {}, resources = []) {
|
|
19657
|
+
super(config);
|
|
19658
|
+
this.resources = resources;
|
|
19659
|
+
this.connectionString = config.connectionString;
|
|
19660
|
+
this.region = config.region;
|
|
19661
|
+
this.bucket = config.bucket;
|
|
19662
|
+
this.keyPrefix = config.keyPrefix;
|
|
19663
|
+
}
|
|
19664
|
+
validateConfig() {
|
|
19665
|
+
const errors = [];
|
|
19666
|
+
if (!this.connectionString && !this.bucket) {
|
|
19667
|
+
errors.push("Either connectionString or bucket must be provided");
|
|
19668
|
+
}
|
|
19669
|
+
return {
|
|
19670
|
+
isValid: errors.length === 0,
|
|
19671
|
+
errors
|
|
19672
|
+
};
|
|
19673
|
+
}
|
|
19674
|
+
async initialize(database) {
|
|
19675
|
+
await super.initialize(database);
|
|
19676
|
+
const targetConfig = {
|
|
19677
|
+
connectionString: this.connectionString,
|
|
19678
|
+
region: this.region,
|
|
19679
|
+
bucket: this.bucket,
|
|
19680
|
+
keyPrefix: this.keyPrefix,
|
|
19681
|
+
verbose: this.config.verbose || false
|
|
19682
|
+
};
|
|
19683
|
+
this.targetDatabase = new S3db(targetConfig);
|
|
19684
|
+
await this.targetDatabase.connect();
|
|
19685
|
+
this.emit("connected", {
|
|
19686
|
+
replicator: this.name,
|
|
19687
|
+
target: this.connectionString || this.bucket
|
|
19688
|
+
});
|
|
19689
|
+
}
|
|
19690
|
+
async replicate(resourceName, operation, data, id) {
|
|
19691
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19692
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19693
|
+
}
|
|
19694
|
+
try {
|
|
19695
|
+
let result;
|
|
19696
|
+
switch (operation) {
|
|
19697
|
+
case "insert":
|
|
19698
|
+
result = await this.targetDatabase.resources[resourceName]?.insert(data);
|
|
19699
|
+
break;
|
|
19700
|
+
case "update":
|
|
19701
|
+
result = await this.targetDatabase.resources[resourceName]?.update(id, data);
|
|
19702
|
+
break;
|
|
19703
|
+
case "delete":
|
|
19704
|
+
result = await this.targetDatabase.resources[resourceName]?.delete(id);
|
|
19705
|
+
break;
|
|
19706
|
+
default:
|
|
19707
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
19708
|
+
}
|
|
19709
|
+
this.emit("replicated", {
|
|
19710
|
+
replicator: this.name,
|
|
19711
|
+
resourceName,
|
|
19712
|
+
operation,
|
|
19713
|
+
id,
|
|
19714
|
+
success: true
|
|
19715
|
+
});
|
|
19716
|
+
return { success: true, result };
|
|
19717
|
+
} catch (error) {
|
|
19718
|
+
this.emit("replication_error", {
|
|
19719
|
+
replicator: this.name,
|
|
19720
|
+
resourceName,
|
|
19721
|
+
operation,
|
|
19722
|
+
id,
|
|
19723
|
+
error: error.message
|
|
19724
|
+
});
|
|
19725
|
+
return { success: false, error: error.message };
|
|
19726
|
+
}
|
|
19727
|
+
}
|
|
19728
|
+
async replicateBatch(resourceName, records) {
|
|
19729
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19730
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19731
|
+
}
|
|
19732
|
+
try {
|
|
19733
|
+
const results = [];
|
|
19734
|
+
const errors = [];
|
|
19735
|
+
for (const record of records) {
|
|
19736
|
+
try {
|
|
19737
|
+
const result = await this.replicate(
|
|
19738
|
+
resourceName,
|
|
19739
|
+
record.operation,
|
|
19740
|
+
record.data,
|
|
19741
|
+
record.id
|
|
19742
|
+
);
|
|
19743
|
+
results.push(result);
|
|
19744
|
+
} catch (error) {
|
|
19745
|
+
errors.push({ id: record.id, error: error.message });
|
|
19746
|
+
}
|
|
19747
|
+
}
|
|
19748
|
+
this.emit("batch_replicated", {
|
|
19749
|
+
replicator: this.name,
|
|
19750
|
+
resourceName,
|
|
19751
|
+
total: records.length,
|
|
19752
|
+
successful: results.filter((r) => r.success).length,
|
|
19753
|
+
errors: errors.length
|
|
19754
|
+
});
|
|
19755
|
+
return {
|
|
19756
|
+
success: errors.length === 0,
|
|
19757
|
+
results,
|
|
19758
|
+
errors,
|
|
19759
|
+
total: records.length
|
|
19760
|
+
};
|
|
19761
|
+
} catch (error) {
|
|
19762
|
+
this.emit("batch_replication_error", {
|
|
19763
|
+
replicator: this.name,
|
|
19764
|
+
resourceName,
|
|
19765
|
+
error: error.message
|
|
19766
|
+
});
|
|
19767
|
+
return { success: false, error: error.message };
|
|
19768
|
+
}
|
|
19769
|
+
}
|
|
19770
|
+
async testConnection() {
|
|
19771
|
+
try {
|
|
19772
|
+
if (!this.targetDatabase) {
|
|
19773
|
+
await this.initialize(this.database);
|
|
19774
|
+
}
|
|
19775
|
+
await this.targetDatabase.listResources();
|
|
19776
|
+
return true;
|
|
19777
|
+
} catch (error) {
|
|
19778
|
+
this.emit("connection_error", {
|
|
19779
|
+
replicator: this.name,
|
|
19780
|
+
error: error.message
|
|
19781
|
+
});
|
|
19782
|
+
return false;
|
|
19783
|
+
}
|
|
19784
|
+
}
|
|
19785
|
+
async getStatus() {
|
|
19786
|
+
const baseStatus = await super.getStatus();
|
|
19787
|
+
return {
|
|
19788
|
+
...baseStatus,
|
|
19789
|
+
connected: !!this.targetDatabase,
|
|
19790
|
+
targetDatabase: this.connectionString || this.bucket,
|
|
19791
|
+
resources: this.resources,
|
|
19792
|
+
totalReplications: this.listenerCount("replicated"),
|
|
19793
|
+
totalErrors: this.listenerCount("replication_error")
|
|
19794
|
+
};
|
|
19795
|
+
}
|
|
19796
|
+
async cleanup() {
|
|
19797
|
+
if (this.targetDatabase) {
|
|
19798
|
+
this.targetDatabase.removeAllListeners();
|
|
19799
|
+
}
|
|
19800
|
+
await super.cleanup();
|
|
19801
|
+
}
|
|
19802
|
+
shouldReplicateResource(resourceName) {
|
|
19803
|
+
return this.resources.length === 0 || this.resources.includes(resourceName);
|
|
19804
|
+
}
|
|
19805
|
+
}
|
|
19806
|
+
|
|
19807
|
+
class SqsReplicator extends BaseReplicator {
|
|
19808
|
+
constructor(config = {}, resources = []) {
|
|
19809
|
+
super(config);
|
|
19810
|
+
this.resources = resources;
|
|
19811
|
+
this.queueUrl = config.queueUrl;
|
|
19812
|
+
this.queues = config.queues || {};
|
|
19813
|
+
this.defaultQueueUrl = config.defaultQueueUrl;
|
|
19814
|
+
this.region = config.region || "us-east-1";
|
|
19815
|
+
this.sqsClient = null;
|
|
19816
|
+
this.messageGroupId = config.messageGroupId;
|
|
19817
|
+
this.deduplicationId = config.deduplicationId;
|
|
19818
|
+
}
|
|
19819
|
+
validateConfig() {
|
|
19820
|
+
const errors = [];
|
|
19821
|
+
if (!this.queueUrl && Object.keys(this.queues).length === 0 && !this.defaultQueueUrl) {
|
|
19822
|
+
errors.push("Either queueUrl, queues object, or defaultQueueUrl must be provided");
|
|
19823
|
+
}
|
|
19824
|
+
return {
|
|
19825
|
+
isValid: errors.length === 0,
|
|
19826
|
+
errors
|
|
19827
|
+
};
|
|
19828
|
+
}
|
|
19829
|
+
/**
|
|
19830
|
+
* Get the appropriate queue URL for a resource
|
|
19831
|
+
*/
|
|
19832
|
+
getQueueUrlForResource(resourceName) {
|
|
19833
|
+
if (this.queues[resourceName]) {
|
|
19834
|
+
return this.queues[resourceName];
|
|
19835
|
+
}
|
|
19836
|
+
if (this.queueUrl) {
|
|
19837
|
+
return this.queueUrl;
|
|
19838
|
+
}
|
|
19839
|
+
if (this.defaultQueueUrl) {
|
|
19840
|
+
return this.defaultQueueUrl;
|
|
19841
|
+
}
|
|
19842
|
+
throw new Error(`No queue URL found for resource '${resourceName}'`);
|
|
19843
|
+
}
|
|
19844
|
+
/**
|
|
19845
|
+
* Create standardized message structure
|
|
19846
|
+
*/
|
|
19847
|
+
createMessage(resourceName, operation, data, id, beforeData = null) {
|
|
19848
|
+
const baseMessage = {
|
|
19849
|
+
resource: resourceName,
|
|
19850
|
+
action: operation,
|
|
19851
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19852
|
+
source: "s3db-replication"
|
|
19853
|
+
};
|
|
19854
|
+
switch (operation) {
|
|
19855
|
+
case "insert":
|
|
19856
|
+
return {
|
|
19857
|
+
...baseMessage,
|
|
19858
|
+
data
|
|
19859
|
+
};
|
|
19860
|
+
case "update":
|
|
19861
|
+
return {
|
|
19862
|
+
...baseMessage,
|
|
19863
|
+
before: beforeData,
|
|
19864
|
+
data
|
|
19865
|
+
};
|
|
19866
|
+
case "delete":
|
|
19867
|
+
return {
|
|
19868
|
+
...baseMessage,
|
|
19869
|
+
data
|
|
19870
|
+
};
|
|
19871
|
+
default:
|
|
19872
|
+
return {
|
|
19873
|
+
...baseMessage,
|
|
19874
|
+
data
|
|
19875
|
+
};
|
|
19876
|
+
}
|
|
19877
|
+
}
|
|
19878
|
+
async initialize(database) {
|
|
19879
|
+
await super.initialize(database);
|
|
19880
|
+
try {
|
|
19881
|
+
const { SQSClient, SendMessageCommand, SendMessageBatchCommand } = await import('@aws-sdk/client-sqs');
|
|
19882
|
+
this.sqsClient = new SQSClient({
|
|
19883
|
+
region: this.region,
|
|
19884
|
+
credentials: this.config.credentials
|
|
19885
|
+
});
|
|
19886
|
+
this.emit("initialized", {
|
|
19887
|
+
replicator: this.name,
|
|
19888
|
+
queueUrl: this.queueUrl,
|
|
19889
|
+
queues: this.queues,
|
|
19890
|
+
defaultQueueUrl: this.defaultQueueUrl
|
|
19891
|
+
});
|
|
19892
|
+
} catch (error) {
|
|
19893
|
+
this.emit("initialization_error", {
|
|
19894
|
+
replicator: this.name,
|
|
19895
|
+
error: error.message
|
|
19896
|
+
});
|
|
19897
|
+
throw error;
|
|
19898
|
+
}
|
|
19899
|
+
}
|
|
19900
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
19901
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19902
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19903
|
+
}
|
|
19904
|
+
try {
|
|
19905
|
+
const { SendMessageCommand } = await import('@aws-sdk/client-sqs');
|
|
19906
|
+
const queueUrl = this.getQueueUrlForResource(resourceName);
|
|
19907
|
+
const message = this.createMessage(resourceName, operation, data, id, beforeData);
|
|
19908
|
+
const command = new SendMessageCommand({
|
|
19909
|
+
QueueUrl: queueUrl,
|
|
19910
|
+
MessageBody: JSON.stringify(message),
|
|
19911
|
+
MessageGroupId: this.messageGroupId,
|
|
19912
|
+
MessageDeduplicationId: this.deduplicationId ? `${resourceName}:${operation}:${id}` : void 0
|
|
19913
|
+
});
|
|
19914
|
+
const result = await this.sqsClient.send(command);
|
|
19915
|
+
this.emit("replicated", {
|
|
19916
|
+
replicator: this.name,
|
|
19917
|
+
resourceName,
|
|
19918
|
+
operation,
|
|
19919
|
+
id,
|
|
19920
|
+
queueUrl,
|
|
19921
|
+
messageId: result.MessageId,
|
|
19922
|
+
success: true
|
|
19923
|
+
});
|
|
19924
|
+
return { success: true, messageId: result.MessageId, queueUrl };
|
|
19925
|
+
} catch (error) {
|
|
19926
|
+
this.emit("replication_error", {
|
|
19927
|
+
replicator: this.name,
|
|
19928
|
+
resourceName,
|
|
19929
|
+
operation,
|
|
19930
|
+
id,
|
|
19931
|
+
error: error.message
|
|
19932
|
+
});
|
|
19933
|
+
return { success: false, error: error.message };
|
|
19934
|
+
}
|
|
19935
|
+
}
|
|
19936
|
+
async replicateBatch(resourceName, records) {
|
|
19937
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
19938
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
19939
|
+
}
|
|
19940
|
+
try {
|
|
19941
|
+
const { SendMessageBatchCommand } = await import('@aws-sdk/client-sqs');
|
|
19942
|
+
const queueUrl = this.getQueueUrlForResource(resourceName);
|
|
19943
|
+
const batchSize = 10;
|
|
19944
|
+
const batches = [];
|
|
19945
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
19946
|
+
batches.push(records.slice(i, i + batchSize));
|
|
19947
|
+
}
|
|
19948
|
+
const results = [];
|
|
19949
|
+
const errors = [];
|
|
19950
|
+
for (const batch of batches) {
|
|
19951
|
+
try {
|
|
19952
|
+
const entries = batch.map((record, index) => ({
|
|
19953
|
+
Id: `${record.id}-${index}`,
|
|
19954
|
+
MessageBody: JSON.stringify(this.createMessage(
|
|
19955
|
+
resourceName,
|
|
19956
|
+
record.operation,
|
|
19957
|
+
record.data,
|
|
19958
|
+
record.id,
|
|
19959
|
+
record.beforeData
|
|
19960
|
+
)),
|
|
19961
|
+
MessageGroupId: this.messageGroupId,
|
|
19962
|
+
MessageDeduplicationId: this.deduplicationId ? `${resourceName}:${record.operation}:${record.id}` : void 0
|
|
19963
|
+
}));
|
|
19964
|
+
const command = new SendMessageBatchCommand({
|
|
19965
|
+
QueueUrl: queueUrl,
|
|
19966
|
+
Entries: entries
|
|
19967
|
+
});
|
|
19968
|
+
const result = await this.sqsClient.send(command);
|
|
19969
|
+
results.push(result);
|
|
19970
|
+
} catch (error) {
|
|
19971
|
+
errors.push({ batch: batch.length, error: error.message });
|
|
19972
|
+
}
|
|
19973
|
+
}
|
|
19974
|
+
this.emit("batch_replicated", {
|
|
19975
|
+
replicator: this.name,
|
|
19976
|
+
resourceName,
|
|
19977
|
+
queueUrl,
|
|
19978
|
+
total: records.length,
|
|
19979
|
+
successful: results.length,
|
|
19980
|
+
errors: errors.length
|
|
19981
|
+
});
|
|
19982
|
+
return {
|
|
19983
|
+
success: errors.length === 0,
|
|
19984
|
+
results,
|
|
19985
|
+
errors,
|
|
19986
|
+
total: records.length,
|
|
19987
|
+
queueUrl
|
|
19988
|
+
};
|
|
19989
|
+
} catch (error) {
|
|
19990
|
+
this.emit("batch_replication_error", {
|
|
19991
|
+
replicator: this.name,
|
|
19992
|
+
resourceName,
|
|
19993
|
+
error: error.message
|
|
19994
|
+
});
|
|
19995
|
+
return { success: false, error: error.message };
|
|
19996
|
+
}
|
|
19997
|
+
}
|
|
19998
|
+
async testConnection() {
|
|
19999
|
+
try {
|
|
20000
|
+
if (!this.sqsClient) {
|
|
20001
|
+
await this.initialize(this.database);
|
|
20002
|
+
}
|
|
20003
|
+
const { GetQueueAttributesCommand } = await import('@aws-sdk/client-sqs');
|
|
20004
|
+
const command = new GetQueueAttributesCommand({
|
|
20005
|
+
QueueUrl: this.queueUrl,
|
|
20006
|
+
AttributeNames: ["QueueArn"]
|
|
20007
|
+
});
|
|
20008
|
+
await this.sqsClient.send(command);
|
|
20009
|
+
return true;
|
|
20010
|
+
} catch (error) {
|
|
20011
|
+
this.emit("connection_error", {
|
|
20012
|
+
replicator: this.name,
|
|
20013
|
+
error: error.message
|
|
20014
|
+
});
|
|
20015
|
+
return false;
|
|
20016
|
+
}
|
|
20017
|
+
}
|
|
20018
|
+
async getStatus() {
|
|
20019
|
+
const baseStatus = await super.getStatus();
|
|
20020
|
+
return {
|
|
20021
|
+
...baseStatus,
|
|
20022
|
+
connected: !!this.sqsClient,
|
|
20023
|
+
queueUrl: this.queueUrl,
|
|
20024
|
+
region: this.region,
|
|
20025
|
+
resources: this.resources,
|
|
20026
|
+
totalReplications: this.listenerCount("replicated"),
|
|
20027
|
+
totalErrors: this.listenerCount("replication_error")
|
|
20028
|
+
};
|
|
20029
|
+
}
|
|
20030
|
+
async cleanup() {
|
|
20031
|
+
if (this.sqsClient) {
|
|
20032
|
+
this.sqsClient.destroy();
|
|
20033
|
+
}
|
|
20034
|
+
await super.cleanup();
|
|
20035
|
+
}
|
|
20036
|
+
shouldReplicateResource(resourceName) {
|
|
20037
|
+
return this.resources.length === 0 || this.resources.includes(resourceName);
|
|
20038
|
+
}
|
|
20039
|
+
}
|
|
20040
|
+
|
|
20041
|
+
class BigqueryReplicator extends BaseReplicator {
|
|
20042
|
+
constructor(config = {}, resources = []) {
|
|
20043
|
+
super(config);
|
|
20044
|
+
this.resources = resources;
|
|
20045
|
+
this.projectId = config.projectId;
|
|
20046
|
+
this.datasetId = config.datasetId;
|
|
20047
|
+
this.tableId = config.tableId;
|
|
20048
|
+
this.tableMap = config.tableMap || {};
|
|
20049
|
+
this.bigqueryClient = null;
|
|
20050
|
+
this.credentials = config.credentials;
|
|
20051
|
+
this.location = config.location || "US";
|
|
20052
|
+
this.logOperations = config.logOperations !== false;
|
|
20053
|
+
}
|
|
20054
|
+
validateConfig() {
|
|
20055
|
+
const errors = [];
|
|
20056
|
+
if (!this.projectId) errors.push("projectId is required");
|
|
20057
|
+
if (!this.datasetId) errors.push("datasetId is required");
|
|
20058
|
+
if (!this.tableId) errors.push("tableId is required");
|
|
20059
|
+
return { isValid: errors.length === 0, errors };
|
|
20060
|
+
}
|
|
20061
|
+
async initialize(database) {
|
|
20062
|
+
await super.initialize(database);
|
|
20063
|
+
try {
|
|
20064
|
+
const { BigQuery } = await import('@google-cloud/bigquery');
|
|
20065
|
+
this.bigqueryClient = new BigQuery({
|
|
20066
|
+
projectId: this.projectId,
|
|
20067
|
+
credentials: this.credentials,
|
|
20068
|
+
location: this.location
|
|
20069
|
+
});
|
|
20070
|
+
this.emit("initialized", {
|
|
20071
|
+
replicator: this.name,
|
|
20072
|
+
projectId: this.projectId,
|
|
20073
|
+
datasetId: this.datasetId,
|
|
20074
|
+
tableId: this.tableId
|
|
20075
|
+
});
|
|
20076
|
+
} catch (error) {
|
|
20077
|
+
this.emit("initialization_error", { replicator: this.name, error: error.message });
|
|
20078
|
+
throw error;
|
|
20079
|
+
}
|
|
20080
|
+
}
|
|
20081
|
+
getTableForResource(resourceName) {
|
|
20082
|
+
return this.tableMap[resourceName] || this.tableId;
|
|
20083
|
+
}
|
|
20084
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
20085
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
20086
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
20087
|
+
}
|
|
20088
|
+
try {
|
|
20089
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
20090
|
+
const tableId = this.getTableForResource(resourceName);
|
|
20091
|
+
const table = dataset.table(tableId);
|
|
20092
|
+
let job;
|
|
20093
|
+
if (operation === "insert") {
|
|
20094
|
+
const row = { ...data };
|
|
20095
|
+
job = await table.insert([row]);
|
|
20096
|
+
} else if (operation === "update") {
|
|
20097
|
+
const keys = Object.keys(data).filter((k) => k !== "id");
|
|
20098
|
+
const setClause = keys.map((k) => `
|
|
20099
|
+
${k}=@${k}
|
|
20100
|
+
`).join(", ");
|
|
20101
|
+
const params = { id };
|
|
20102
|
+
keys.forEach((k) => {
|
|
20103
|
+
params[k] = data[k];
|
|
20104
|
+
});
|
|
20105
|
+
const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableId}\`
|
|
20106
|
+
SET ${setClause}
|
|
20107
|
+
WHERE id=@id`;
|
|
20108
|
+
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
20109
|
+
query,
|
|
20110
|
+
params
|
|
20111
|
+
});
|
|
20112
|
+
await updateJob.getQueryResults();
|
|
20113
|
+
job = [updateJob];
|
|
20114
|
+
} else if (operation === "delete") {
|
|
20115
|
+
const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableId}\`
|
|
20116
|
+
WHERE id=@id`;
|
|
20117
|
+
const [deleteJob] = await this.bigqueryClient.createQueryJob({
|
|
20118
|
+
query,
|
|
20119
|
+
params: { id }
|
|
20120
|
+
});
|
|
20121
|
+
await deleteJob.getQueryResults();
|
|
20122
|
+
job = [deleteJob];
|
|
20123
|
+
} else {
|
|
20124
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
20125
|
+
}
|
|
20126
|
+
if (this.logOperations) {
|
|
20127
|
+
const logTable = dataset.table(this.tableId);
|
|
20128
|
+
await logTable.insert([{
|
|
20129
|
+
resource_name: resourceName,
|
|
20130
|
+
operation,
|
|
20131
|
+
record_id: id,
|
|
20132
|
+
data: JSON.stringify(data),
|
|
20133
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20134
|
+
source: "s3db-replication"
|
|
20135
|
+
}]);
|
|
20136
|
+
}
|
|
20137
|
+
this.emit("replicated", {
|
|
20138
|
+
replicator: this.name,
|
|
20139
|
+
resourceName,
|
|
20140
|
+
operation,
|
|
20141
|
+
id,
|
|
20142
|
+
jobId: job[0]?.id,
|
|
20143
|
+
success: true
|
|
20144
|
+
});
|
|
20145
|
+
return { success: true, jobId: job[0]?.id };
|
|
20146
|
+
} catch (error) {
|
|
20147
|
+
this.emit("replication_error", {
|
|
20148
|
+
replicator: this.name,
|
|
20149
|
+
resourceName,
|
|
20150
|
+
operation,
|
|
20151
|
+
id,
|
|
20152
|
+
error: error.message
|
|
20153
|
+
});
|
|
20154
|
+
return { success: false, error: error.message };
|
|
20155
|
+
}
|
|
20156
|
+
}
|
|
20157
|
+
async replicateBatch(resourceName, records) {
|
|
20158
|
+
const results = [];
|
|
20159
|
+
const errors = [];
|
|
20160
|
+
for (const record of records) {
|
|
20161
|
+
try {
|
|
20162
|
+
const res = await this.replicate(resourceName, record.operation, record.data, record.id, record.beforeData);
|
|
20163
|
+
results.push(res);
|
|
20164
|
+
} catch (err) {
|
|
20165
|
+
errors.push({ id: record.id, error: err.message });
|
|
20166
|
+
}
|
|
20167
|
+
}
|
|
20168
|
+
return { success: errors.length === 0, results, errors };
|
|
20169
|
+
}
|
|
20170
|
+
async testConnection() {
|
|
20171
|
+
try {
|
|
20172
|
+
if (!this.bigqueryClient) await this.initialize();
|
|
20173
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
20174
|
+
await dataset.getMetadata();
|
|
20175
|
+
return true;
|
|
20176
|
+
} catch (error) {
|
|
20177
|
+
this.emit("connection_error", { replicator: this.name, error: error.message });
|
|
20178
|
+
return false;
|
|
20179
|
+
}
|
|
20180
|
+
}
|
|
20181
|
+
async cleanup() {
|
|
20182
|
+
}
|
|
20183
|
+
shouldReplicateResource(resourceName) {
|
|
20184
|
+
if (!this.resources || this.resources.length === 0) return true;
|
|
20185
|
+
return this.resources.includes(resourceName);
|
|
20186
|
+
}
|
|
20187
|
+
}
|
|
20188
|
+
|
|
20189
|
+
class PostgresReplicator extends BaseReplicator {
|
|
20190
|
+
constructor(config = {}, resources = []) {
|
|
20191
|
+
super(config);
|
|
20192
|
+
this.resources = resources;
|
|
20193
|
+
this.connectionString = config.connectionString;
|
|
20194
|
+
this.host = config.host;
|
|
20195
|
+
this.port = config.port || 5432;
|
|
20196
|
+
this.database = config.database;
|
|
20197
|
+
this.user = config.user;
|
|
20198
|
+
this.password = config.password;
|
|
20199
|
+
this.tableName = config.tableName || "s3db_replication";
|
|
20200
|
+
this.tableMap = config.tableMap || {};
|
|
20201
|
+
this.client = null;
|
|
20202
|
+
this.ssl = config.ssl;
|
|
20203
|
+
this.logOperations = config.logOperations !== false;
|
|
20204
|
+
}
|
|
20205
|
+
validateConfig() {
|
|
20206
|
+
const errors = [];
|
|
20207
|
+
if (!this.connectionString && (!this.host || !this.database)) {
|
|
20208
|
+
errors.push("Either connectionString or host+database must be provided");
|
|
20209
|
+
}
|
|
20210
|
+
return {
|
|
20211
|
+
isValid: errors.length === 0,
|
|
20212
|
+
errors
|
|
20213
|
+
};
|
|
20214
|
+
}
|
|
20215
|
+
async initialize(database) {
|
|
20216
|
+
await super.initialize(database);
|
|
20217
|
+
try {
|
|
20218
|
+
const { Client } = await import('pg');
|
|
20219
|
+
const config = this.connectionString ? {
|
|
20220
|
+
connectionString: this.connectionString,
|
|
20221
|
+
ssl: this.ssl
|
|
20222
|
+
} : {
|
|
20223
|
+
host: this.host,
|
|
20224
|
+
port: this.port,
|
|
20225
|
+
database: this.database,
|
|
20226
|
+
user: this.user,
|
|
20227
|
+
password: this.password,
|
|
20228
|
+
ssl: this.ssl
|
|
20229
|
+
};
|
|
20230
|
+
this.client = new Client(config);
|
|
20231
|
+
await this.client.connect();
|
|
20232
|
+
if (this.logOperations) await this.createTableIfNotExists();
|
|
20233
|
+
this.emit("initialized", {
|
|
20234
|
+
replicator: this.name,
|
|
20235
|
+
database: this.database || "postgres",
|
|
20236
|
+
table: this.tableName
|
|
20237
|
+
});
|
|
20238
|
+
} catch (error) {
|
|
20239
|
+
this.emit("initialization_error", {
|
|
20240
|
+
replicator: this.name,
|
|
20241
|
+
error: error.message
|
|
20242
|
+
});
|
|
20243
|
+
throw error;
|
|
20244
|
+
}
|
|
20245
|
+
}
|
|
20246
|
+
async createTableIfNotExists() {
|
|
20247
|
+
const createTableQuery = `
|
|
20248
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
20249
|
+
id SERIAL PRIMARY KEY,
|
|
20250
|
+
resource_name VARCHAR(255) NOT NULL,
|
|
20251
|
+
operation VARCHAR(50) NOT NULL,
|
|
20252
|
+
record_id VARCHAR(255) NOT NULL,
|
|
20253
|
+
data JSONB,
|
|
20254
|
+
timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
20255
|
+
source VARCHAR(100) DEFAULT 's3db-replication',
|
|
20256
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
20257
|
+
);
|
|
20258
|
+
CREATE INDEX IF NOT EXISTS idx_${this.tableName}_resource_name ON ${this.tableName}(resource_name);
|
|
20259
|
+
CREATE INDEX IF NOT EXISTS idx_${this.tableName}_operation ON ${this.tableName}(operation);
|
|
20260
|
+
CREATE INDEX IF NOT EXISTS idx_${this.tableName}_record_id ON ${this.tableName}(record_id);
|
|
20261
|
+
CREATE INDEX IF NOT EXISTS idx_${this.tableName}_timestamp ON ${this.tableName}(timestamp);
|
|
20262
|
+
`;
|
|
20263
|
+
await this.client.query(createTableQuery);
|
|
20264
|
+
}
|
|
20265
|
+
getTableForResource(resourceName) {
|
|
20266
|
+
return this.tableMap[resourceName] || resourceName;
|
|
20267
|
+
}
|
|
20268
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
20269
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
20270
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
20271
|
+
}
|
|
20272
|
+
try {
|
|
20273
|
+
const table = this.getTableForResource(resourceName);
|
|
20274
|
+
let result;
|
|
20275
|
+
if (operation === "insert") {
|
|
20276
|
+
const keys = Object.keys(data);
|
|
20277
|
+
const values = keys.map((k) => data[k]);
|
|
20278
|
+
const columns = keys.map((k) => `"${k}"`).join(", ");
|
|
20279
|
+
const params = keys.map((_, i) => `$${i + 1}`).join(", ");
|
|
20280
|
+
const sql = `INSERT INTO ${table} (${columns}) VALUES (${params}) ON CONFLICT (id) DO NOTHING RETURNING *`;
|
|
20281
|
+
result = await this.client.query(sql, values);
|
|
20282
|
+
} else if (operation === "update") {
|
|
20283
|
+
const keys = Object.keys(data).filter((k) => k !== "id");
|
|
20284
|
+
const setClause = keys.map((k, i) => `"${k}"=$${i + 1}`).join(", ");
|
|
20285
|
+
const values = keys.map((k) => data[k]);
|
|
20286
|
+
values.push(id);
|
|
20287
|
+
const sql = `UPDATE ${table} SET ${setClause} WHERE id=$${keys.length + 1} RETURNING *`;
|
|
20288
|
+
result = await this.client.query(sql, values);
|
|
20289
|
+
} else if (operation === "delete") {
|
|
20290
|
+
const sql = `DELETE FROM ${table} WHERE id=$1 RETURNING *`;
|
|
20291
|
+
result = await this.client.query(sql, [id]);
|
|
20292
|
+
} else {
|
|
20293
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
20294
|
+
}
|
|
20295
|
+
if (this.logOperations) {
|
|
20296
|
+
await this.client.query(
|
|
20297
|
+
`INSERT INTO ${this.tableName} (resource_name, operation, record_id, data, timestamp, source) VALUES ($1, $2, $3, $4, $5, $6)`,
|
|
20298
|
+
[resourceName, operation, id, JSON.stringify(data), (/* @__PURE__ */ new Date()).toISOString(), "s3db-replication"]
|
|
20299
|
+
);
|
|
20300
|
+
}
|
|
20301
|
+
this.emit("replicated", {
|
|
20302
|
+
replicator: this.name,
|
|
20303
|
+
resourceName,
|
|
20304
|
+
operation,
|
|
20305
|
+
id,
|
|
20306
|
+
result: result.rows,
|
|
20307
|
+
success: true
|
|
20308
|
+
});
|
|
20309
|
+
return { success: true, rows: result.rows };
|
|
20310
|
+
} catch (error) {
|
|
20311
|
+
this.emit("replication_error", {
|
|
20312
|
+
replicator: this.name,
|
|
20313
|
+
resourceName,
|
|
20314
|
+
operation,
|
|
20315
|
+
id,
|
|
20316
|
+
error: error.message
|
|
20317
|
+
});
|
|
20318
|
+
return { success: false, error: error.message };
|
|
20319
|
+
}
|
|
20320
|
+
}
|
|
20321
|
+
async replicateBatch(resourceName, records) {
|
|
20322
|
+
const results = [];
|
|
20323
|
+
const errors = [];
|
|
20324
|
+
for (const record of records) {
|
|
20325
|
+
try {
|
|
20326
|
+
const res = await this.replicate(resourceName, record.operation, record.data, record.id, record.beforeData);
|
|
20327
|
+
results.push(res);
|
|
20328
|
+
} catch (err) {
|
|
20329
|
+
errors.push({ id: record.id, error: err.message });
|
|
20330
|
+
}
|
|
20331
|
+
}
|
|
20332
|
+
return { success: errors.length === 0, results, errors };
|
|
20333
|
+
}
|
|
20334
|
+
async testConnection() {
|
|
20335
|
+
try {
|
|
20336
|
+
if (!this.client) await this.initialize();
|
|
20337
|
+
await this.client.query("SELECT 1");
|
|
20338
|
+
return true;
|
|
20339
|
+
} catch (error) {
|
|
20340
|
+
this.emit("connection_error", { replicator: this.name, error: error.message });
|
|
20341
|
+
return false;
|
|
20342
|
+
}
|
|
20343
|
+
}
|
|
20344
|
+
async cleanup() {
|
|
20345
|
+
if (this.client) await this.client.end();
|
|
20346
|
+
}
|
|
20347
|
+
shouldReplicateResource(resourceName) {
|
|
20348
|
+
if (!this.resources || this.resources.length === 0) return true;
|
|
20349
|
+
return this.resources.includes(resourceName);
|
|
20350
|
+
}
|
|
20351
|
+
}
|
|
20352
|
+
|
|
20353
|
+
const REPLICATOR_DRIVERS = {
|
|
20354
|
+
s3db: S3dbReplicator,
|
|
20355
|
+
sqs: SqsReplicator,
|
|
20356
|
+
bigquery: BigqueryReplicator,
|
|
20357
|
+
postgres: PostgresReplicator
|
|
20358
|
+
};
|
|
20359
|
+
function createReplicator(driver, config = {}, resources = []) {
|
|
20360
|
+
const ReplicatorClass = REPLICATOR_DRIVERS[driver];
|
|
20361
|
+
if (!ReplicatorClass) {
|
|
20362
|
+
throw new Error(`Unknown replicator driver: ${driver}. Available drivers: ${Object.keys(REPLICATOR_DRIVERS).join(", ")}`);
|
|
20363
|
+
}
|
|
20364
|
+
return new ReplicatorClass(config, resources);
|
|
20365
|
+
}
|
|
20366
|
+
function validateReplicatorConfig(driver, config, resources = []) {
|
|
20367
|
+
const replicator = createReplicator(driver, config, resources);
|
|
20368
|
+
return replicator.validateConfig();
|
|
20369
|
+
}
|
|
20370
|
+
|
|
20371
|
+
const gzipAsync = promisify(gzip);
|
|
20372
|
+
const gunzipAsync = promisify(gunzip);
|
|
20373
|
+
class ReplicationPlugin extends Plugin {
|
|
20374
|
+
constructor(options = {}) {
|
|
20375
|
+
super();
|
|
20376
|
+
this.config = {
|
|
20377
|
+
enabled: options.enabled !== false,
|
|
20378
|
+
replicators: options.replicators || [],
|
|
20379
|
+
syncMode: options.syncMode || "async",
|
|
20380
|
+
// 'sync' or 'async'
|
|
20381
|
+
retryAttempts: options.retryAttempts || 3,
|
|
20382
|
+
retryDelay: options.retryDelay || 1e3,
|
|
20383
|
+
// ms
|
|
20384
|
+
batchSize: options.batchSize || 10,
|
|
20385
|
+
compression: options.compression || false,
|
|
20386
|
+
// Enable compression
|
|
20387
|
+
compressionLevel: options.compressionLevel || 6,
|
|
20388
|
+
// 0-9
|
|
20389
|
+
...options
|
|
20390
|
+
};
|
|
20391
|
+
this.replicators = [];
|
|
20392
|
+
this.queue = [];
|
|
20393
|
+
this.isProcessing = false;
|
|
20394
|
+
this.stats = {
|
|
20395
|
+
totalOperations: 0,
|
|
20396
|
+
successfulOperations: 0,
|
|
20397
|
+
failedOperations: 0,
|
|
20398
|
+
lastSync: null
|
|
20399
|
+
};
|
|
20400
|
+
}
|
|
20401
|
+
/**
|
|
20402
|
+
* Process data according to replication mode
|
|
20403
|
+
*/
|
|
20404
|
+
processDataForReplication(data, metadata = {}) {
|
|
20405
|
+
switch (this.config.replicationMode) {
|
|
20406
|
+
case "exact-copy":
|
|
20407
|
+
return {
|
|
20408
|
+
body: data,
|
|
20409
|
+
metadata
|
|
20410
|
+
};
|
|
20411
|
+
case "just-metadata":
|
|
20412
|
+
return {
|
|
20413
|
+
body: null,
|
|
20414
|
+
metadata
|
|
20415
|
+
};
|
|
20416
|
+
case "all-in-body":
|
|
20417
|
+
return {
|
|
20418
|
+
body: {
|
|
20419
|
+
data,
|
|
20420
|
+
metadata,
|
|
20421
|
+
replicationMode: this.config.replicationMode,
|
|
20422
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
20423
|
+
},
|
|
20424
|
+
metadata: {
|
|
20425
|
+
replicationMode: this.config.replicationMode,
|
|
20426
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
20427
|
+
}
|
|
20428
|
+
};
|
|
20429
|
+
default:
|
|
20430
|
+
return {
|
|
20431
|
+
body: data,
|
|
20432
|
+
metadata
|
|
20433
|
+
};
|
|
20434
|
+
}
|
|
20435
|
+
}
|
|
20436
|
+
/**
|
|
20437
|
+
* Compress data if compression is enabled
|
|
20438
|
+
*/
|
|
20439
|
+
async compressData(data) {
|
|
20440
|
+
if (!this.config.compression || !data) {
|
|
20441
|
+
return data;
|
|
20442
|
+
}
|
|
20443
|
+
try {
|
|
20444
|
+
const jsonString = JSON.stringify(data);
|
|
20445
|
+
const compressed = await gzipAsync(jsonString, { level: this.config.compressionLevel });
|
|
20446
|
+
return compressed.toString("base64");
|
|
20447
|
+
} catch (error) {
|
|
20448
|
+
this.emit("replication.compression.failed", { error, data });
|
|
20449
|
+
return data;
|
|
20450
|
+
}
|
|
20451
|
+
}
|
|
20452
|
+
/**
|
|
20453
|
+
* Decompress data if it was compressed
|
|
20454
|
+
*/
|
|
20455
|
+
async decompressData(data) {
|
|
20456
|
+
if (!this.config.compression || !data) {
|
|
20457
|
+
return data;
|
|
20458
|
+
}
|
|
20459
|
+
try {
|
|
20460
|
+
if (typeof data === "string" && data.startsWith("H4sI")) {
|
|
20461
|
+
const buffer = Buffer.from(data, "base64");
|
|
20462
|
+
const decompressed = await gunzipAsync(buffer);
|
|
20463
|
+
return JSON.parse(decompressed.toString());
|
|
20464
|
+
}
|
|
20465
|
+
return data;
|
|
20466
|
+
} catch (error) {
|
|
20467
|
+
this.emit("replication.decompression.failed", { error, data });
|
|
20468
|
+
return data;
|
|
20469
|
+
}
|
|
20470
|
+
}
|
|
20471
|
+
async setup(database) {
|
|
20472
|
+
this.database = database;
|
|
20473
|
+
if (!this.config.enabled) {
|
|
20474
|
+
return;
|
|
20475
|
+
}
|
|
20476
|
+
if (this.config.replicators && this.config.replicators.length > 0) {
|
|
20477
|
+
await this.initializeReplicators();
|
|
20478
|
+
}
|
|
20479
|
+
if (!database.resources.replication_logs) {
|
|
20480
|
+
this.replicationLog = await database.createResource({
|
|
20481
|
+
name: "replication_logs",
|
|
20482
|
+
attributes: {
|
|
20483
|
+
id: "string|required",
|
|
20484
|
+
resourceName: "string|required",
|
|
20485
|
+
operation: "string|required",
|
|
20486
|
+
recordId: "string|required",
|
|
20487
|
+
replicatorId: "string|required",
|
|
20488
|
+
status: "string|required",
|
|
20489
|
+
attempts: "number|required",
|
|
20490
|
+
lastAttempt: "string|required",
|
|
20491
|
+
error: "string|required",
|
|
20492
|
+
data: "object|required",
|
|
20493
|
+
timestamp: "string|required"
|
|
20494
|
+
}
|
|
20495
|
+
});
|
|
20496
|
+
} else {
|
|
20497
|
+
this.replicationLog = database.resources.replication_logs;
|
|
20498
|
+
}
|
|
20499
|
+
for (const resourceName in database.resources) {
|
|
20500
|
+
if (resourceName !== "replication_logs") {
|
|
20501
|
+
this.installHooks(database.resources[resourceName]);
|
|
20502
|
+
}
|
|
20503
|
+
}
|
|
20504
|
+
const originalCreateResource = database.createResource.bind(database);
|
|
20505
|
+
database.createResource = async (config) => {
|
|
20506
|
+
const resource = await originalCreateResource(config);
|
|
20507
|
+
if (resource && resource.name !== "replication_logs") {
|
|
20508
|
+
this.installHooks(resource);
|
|
20509
|
+
}
|
|
20510
|
+
return resource;
|
|
20511
|
+
};
|
|
20512
|
+
this.startQueueProcessor();
|
|
20513
|
+
}
|
|
20514
|
+
async initializeReplicators() {
|
|
20515
|
+
for (const replicatorConfig of this.config.replicators) {
|
|
20516
|
+
try {
|
|
20517
|
+
const { driver, config: replicatorConfigData, resources = [] } = replicatorConfig;
|
|
20518
|
+
const validation = validateReplicatorConfig(driver, replicatorConfigData, resources);
|
|
20519
|
+
if (!validation.isValid) {
|
|
20520
|
+
this.emit("replicator.validation.failed", {
|
|
20521
|
+
driver,
|
|
20522
|
+
errors: validation.errors
|
|
20523
|
+
});
|
|
20524
|
+
continue;
|
|
20525
|
+
}
|
|
20526
|
+
const replicator = createReplicator(driver, replicatorConfigData, resources);
|
|
20527
|
+
await replicator.initialize(this.database);
|
|
20528
|
+
replicator.on("replicated", (data) => {
|
|
20529
|
+
this.emit("replication.success", data);
|
|
20530
|
+
});
|
|
20531
|
+
replicator.on("replication_error", (data) => {
|
|
20532
|
+
this.emit("replication.failed", data);
|
|
20533
|
+
});
|
|
20534
|
+
this.replicators.push({
|
|
20535
|
+
id: `${driver}-${Date.now()}`,
|
|
20536
|
+
driver,
|
|
20537
|
+
config: replicatorConfigData,
|
|
20538
|
+
resources,
|
|
20539
|
+
instance: replicator
|
|
20540
|
+
});
|
|
20541
|
+
this.emit("replicator.initialized", {
|
|
20542
|
+
driver,
|
|
20543
|
+
config: replicatorConfigData,
|
|
20544
|
+
resources
|
|
20545
|
+
});
|
|
20546
|
+
} catch (error) {
|
|
20547
|
+
this.emit("replicator.initialization.failed", {
|
|
20548
|
+
driver: replicatorConfig.driver,
|
|
20549
|
+
error: error.message
|
|
20550
|
+
});
|
|
20551
|
+
}
|
|
20552
|
+
}
|
|
20553
|
+
}
|
|
20554
|
+
async start() {
|
|
20555
|
+
}
|
|
20556
|
+
async stop() {
|
|
20557
|
+
this.isProcessing = false;
|
|
20558
|
+
await this.processQueue();
|
|
20559
|
+
}
|
|
20560
|
+
installHooks(resource) {
|
|
20561
|
+
if (!resource || resource.name === "replication_logs") return;
|
|
20562
|
+
const originalDataMap = /* @__PURE__ */ new Map();
|
|
20563
|
+
resource.addHook("afterInsert", async (data) => {
|
|
20564
|
+
await this.queueReplication(resource.name, "insert", data.id, data);
|
|
20565
|
+
return data;
|
|
20566
|
+
});
|
|
20567
|
+
resource.addHook("preUpdate", async (data) => {
|
|
20568
|
+
if (data.id) {
|
|
20569
|
+
try {
|
|
20570
|
+
const originalData = await resource.get(data.id);
|
|
20571
|
+
originalDataMap.set(data.id, originalData);
|
|
20572
|
+
} catch (error) {
|
|
20573
|
+
originalDataMap.set(data.id, { id: data.id });
|
|
20574
|
+
}
|
|
20575
|
+
}
|
|
20576
|
+
return data;
|
|
20577
|
+
});
|
|
20578
|
+
resource.addHook("afterUpdate", async (data) => {
|
|
20579
|
+
const beforeData = originalDataMap.get(data.id);
|
|
20580
|
+
await this.queueReplication(resource.name, "update", data.id, data, beforeData);
|
|
20581
|
+
originalDataMap.delete(data.id);
|
|
20582
|
+
return data;
|
|
20583
|
+
});
|
|
20584
|
+
resource.addHook("afterDelete", async (data) => {
|
|
20585
|
+
await this.queueReplication(resource.name, "delete", data.id, data);
|
|
20586
|
+
return data;
|
|
20587
|
+
});
|
|
20588
|
+
const originalDeleteMany = resource.deleteMany.bind(resource);
|
|
20589
|
+
resource.deleteMany = async (ids) => {
|
|
20590
|
+
const result = await originalDeleteMany(ids);
|
|
20591
|
+
if (result && result.length > 0) {
|
|
20592
|
+
for (const id of ids) {
|
|
20593
|
+
await this.queueReplication(resource.name, "delete", id, { id });
|
|
20594
|
+
}
|
|
20595
|
+
}
|
|
20596
|
+
return result;
|
|
20597
|
+
};
|
|
20598
|
+
}
|
|
20599
|
+
async queueReplication(resourceName, operation, recordId, data, beforeData = null) {
|
|
20600
|
+
if (!this.config.enabled) {
|
|
20601
|
+
return;
|
|
20602
|
+
}
|
|
20603
|
+
if (this.replicators.length === 0) {
|
|
20604
|
+
return;
|
|
20605
|
+
}
|
|
20606
|
+
const applicableReplicators = this.replicators.filter(
|
|
20607
|
+
(replicator) => replicator.instance.shouldReplicateResource(resourceName)
|
|
20608
|
+
);
|
|
20609
|
+
if (applicableReplicators.length === 0) {
|
|
20610
|
+
return;
|
|
20611
|
+
}
|
|
20612
|
+
const item = {
|
|
20613
|
+
id: `repl-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
20614
|
+
resourceName,
|
|
20615
|
+
operation,
|
|
20616
|
+
recordId,
|
|
20617
|
+
data: isPlainObject(data) ? data : { raw: data },
|
|
20618
|
+
beforeData: beforeData ? isPlainObject(beforeData) ? beforeData : { raw: beforeData } : null,
|
|
20619
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20620
|
+
attempts: 0
|
|
20621
|
+
};
|
|
20622
|
+
const logId = await this.logReplication(item);
|
|
20623
|
+
if (this.config.syncMode === "sync") {
|
|
20624
|
+
try {
|
|
20625
|
+
const result = await this.processReplicationItem(item);
|
|
20626
|
+
if (logId) {
|
|
20627
|
+
await this.updateReplicationLog(logId, {
|
|
20628
|
+
status: result.success ? "success" : "failed",
|
|
20629
|
+
attempts: 1,
|
|
20630
|
+
error: result.success ? "" : JSON.stringify(result.results)
|
|
20631
|
+
});
|
|
20632
|
+
}
|
|
20633
|
+
this.stats.totalOperations++;
|
|
20634
|
+
if (result.success) {
|
|
20635
|
+
this.stats.successfulOperations++;
|
|
20636
|
+
} else {
|
|
20637
|
+
this.stats.failedOperations++;
|
|
20638
|
+
}
|
|
20639
|
+
} catch (error) {
|
|
20640
|
+
if (logId) {
|
|
20641
|
+
await this.updateReplicationLog(logId, {
|
|
20642
|
+
status: "failed",
|
|
20643
|
+
attempts: 1,
|
|
20644
|
+
error: error.message
|
|
20645
|
+
});
|
|
20646
|
+
}
|
|
20647
|
+
this.stats.failedOperations++;
|
|
20648
|
+
}
|
|
20649
|
+
} else {
|
|
20650
|
+
this.queue.push(item);
|
|
20651
|
+
this.emit("replication.queued", { item, queueLength: this.queue.length });
|
|
20652
|
+
}
|
|
20653
|
+
}
|
|
20654
|
+
async processReplicationItem(item) {
|
|
20655
|
+
const { resourceName, operation, recordId, data, beforeData } = item;
|
|
20656
|
+
const applicableReplicators = this.replicators.filter(
|
|
20657
|
+
(replicator) => replicator.instance.shouldReplicateResource(resourceName)
|
|
20658
|
+
);
|
|
20659
|
+
if (applicableReplicators.length === 0) {
|
|
20660
|
+
return { success: true, skipped: true, reason: "no_applicable_replicators" };
|
|
20661
|
+
}
|
|
20662
|
+
const results = [];
|
|
20663
|
+
for (const replicator of applicableReplicators) {
|
|
20664
|
+
try {
|
|
20665
|
+
const result = await replicator.instance.replicate(resourceName, operation, data, recordId, beforeData);
|
|
20666
|
+
results.push({
|
|
20667
|
+
replicatorId: replicator.id,
|
|
20668
|
+
driver: replicator.driver,
|
|
20669
|
+
success: result.success,
|
|
20670
|
+
error: result.error,
|
|
20671
|
+
skipped: result.skipped
|
|
20672
|
+
});
|
|
20673
|
+
} catch (error) {
|
|
20674
|
+
results.push({
|
|
20675
|
+
replicatorId: replicator.id,
|
|
20676
|
+
driver: replicator.driver,
|
|
20677
|
+
success: false,
|
|
20678
|
+
error: error.message
|
|
20679
|
+
});
|
|
20680
|
+
}
|
|
20681
|
+
}
|
|
20682
|
+
return {
|
|
20683
|
+
success: results.every((r) => r.success || r.skipped),
|
|
20684
|
+
results
|
|
20685
|
+
};
|
|
20686
|
+
}
|
|
20687
|
+
async logReplication(item) {
|
|
20688
|
+
if (!this.replicationLog) return;
|
|
20689
|
+
try {
|
|
20690
|
+
const logId = `log-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
20691
|
+
await this.replicationLog.insert({
|
|
20692
|
+
id: logId,
|
|
20693
|
+
resourceName: item.resourceName,
|
|
20694
|
+
operation: item.operation,
|
|
20695
|
+
recordId: item.recordId,
|
|
20696
|
+
replicatorId: "all",
|
|
20697
|
+
// Will be updated with specific replicator results
|
|
20698
|
+
status: "queued",
|
|
20699
|
+
attempts: 0,
|
|
20700
|
+
lastAttempt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20701
|
+
error: "",
|
|
20702
|
+
data: isPlainObject(item.data) ? item.data : { raw: item.data },
|
|
20703
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
20704
|
+
});
|
|
20705
|
+
return logId;
|
|
20706
|
+
} catch (error) {
|
|
20707
|
+
this.emit("replication.log.failed", { error: error.message, item });
|
|
20708
|
+
return null;
|
|
20709
|
+
}
|
|
20710
|
+
}
|
|
20711
|
+
async updateReplicationLog(logId, updates) {
|
|
20712
|
+
if (!this.replicationLog) return;
|
|
20713
|
+
try {
|
|
20714
|
+
await this.replicationLog.update(logId, {
|
|
20715
|
+
...updates,
|
|
20716
|
+
lastAttempt: (/* @__PURE__ */ new Date()).toISOString()
|
|
20717
|
+
});
|
|
20718
|
+
} catch (error) {
|
|
20719
|
+
this.emit("replication.updateLog.failed", { error: error.message, logId, updates });
|
|
20720
|
+
}
|
|
20721
|
+
}
|
|
20722
|
+
startQueueProcessor() {
|
|
20723
|
+
if (this.isProcessing) return;
|
|
20724
|
+
this.isProcessing = true;
|
|
20725
|
+
this.processQueueLoop();
|
|
20726
|
+
}
|
|
20727
|
+
async processQueueLoop() {
|
|
20728
|
+
while (this.isProcessing) {
|
|
20729
|
+
if (this.queue.length > 0) {
|
|
20730
|
+
const batch = this.queue.splice(0, this.config.batchSize);
|
|
20731
|
+
for (const item of batch) {
|
|
20732
|
+
await this.processReplicationItem(item);
|
|
20733
|
+
}
|
|
20734
|
+
} else {
|
|
20735
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
20736
|
+
}
|
|
20737
|
+
}
|
|
20738
|
+
}
|
|
20739
|
+
async processQueue() {
|
|
20740
|
+
if (this.queue.length === 0) return;
|
|
20741
|
+
const item = this.queue.shift();
|
|
20742
|
+
let attempts = 0;
|
|
20743
|
+
let lastError = null;
|
|
20744
|
+
while (attempts < this.config.retryAttempts) {
|
|
20745
|
+
try {
|
|
20746
|
+
attempts++;
|
|
20747
|
+
this.emit("replication.retry.started", {
|
|
20748
|
+
item,
|
|
20749
|
+
attempt: attempts,
|
|
20750
|
+
maxAttempts: this.config.retryAttempts
|
|
20751
|
+
});
|
|
20752
|
+
const result = await this.processReplicationItem(item);
|
|
20753
|
+
if (result.success) {
|
|
20754
|
+
this.stats.successfulOperations++;
|
|
20755
|
+
this.emit("replication.success", {
|
|
20756
|
+
item,
|
|
20757
|
+
attempts,
|
|
20758
|
+
results: result.results,
|
|
20759
|
+
stats: this.stats
|
|
20760
|
+
});
|
|
20761
|
+
return;
|
|
20762
|
+
} else {
|
|
20763
|
+
lastError = result.results;
|
|
20764
|
+
if (attempts < this.config.retryAttempts) {
|
|
20765
|
+
await new Promise((resolve) => setTimeout(resolve, this.config.retryDelay * attempts));
|
|
20766
|
+
}
|
|
20767
|
+
}
|
|
20768
|
+
} catch (error) {
|
|
20769
|
+
lastError = error.message;
|
|
20770
|
+
if (attempts < this.config.retryAttempts) {
|
|
20771
|
+
await new Promise((resolve) => setTimeout(resolve, this.config.retryDelay * attempts));
|
|
20772
|
+
} else {
|
|
20773
|
+
this.emit("replication.retry.exhausted", {
|
|
20774
|
+
attempts,
|
|
20775
|
+
lastError,
|
|
20776
|
+
item
|
|
20777
|
+
});
|
|
20778
|
+
}
|
|
20779
|
+
}
|
|
20780
|
+
}
|
|
20781
|
+
this.stats.failedOperations++;
|
|
20782
|
+
this.emit("replication.failed", {
|
|
20783
|
+
attempts,
|
|
20784
|
+
lastError,
|
|
20785
|
+
item,
|
|
20786
|
+
stats: this.stats
|
|
20787
|
+
});
|
|
20788
|
+
}
|
|
20789
|
+
// Utility methods
|
|
20790
|
+
async getReplicationStats() {
|
|
20791
|
+
const replicatorStats = await Promise.all(
|
|
20792
|
+
this.replicators.map(async (replicator) => {
|
|
20793
|
+
const status = await replicator.instance.getStatus();
|
|
20794
|
+
return {
|
|
20795
|
+
id: replicator.id,
|
|
20796
|
+
driver: replicator.driver,
|
|
20797
|
+
config: replicator.config,
|
|
20798
|
+
status
|
|
20799
|
+
};
|
|
20800
|
+
})
|
|
20801
|
+
);
|
|
20802
|
+
return {
|
|
20803
|
+
enabled: this.config.enabled,
|
|
20804
|
+
replicators: replicatorStats,
|
|
20805
|
+
queue: {
|
|
20806
|
+
length: this.queue.length,
|
|
20807
|
+
isProcessing: this.isProcessing
|
|
20808
|
+
},
|
|
20809
|
+
stats: this.stats,
|
|
20810
|
+
lastSync: this.stats.lastSync
|
|
20811
|
+
};
|
|
20812
|
+
}
|
|
20813
|
+
async getReplicationLogs(options = {}) {
|
|
20814
|
+
if (!this.replicationLog) {
|
|
20815
|
+
return [];
|
|
20816
|
+
}
|
|
20817
|
+
const {
|
|
20818
|
+
resourceName,
|
|
20819
|
+
operation,
|
|
20820
|
+
status,
|
|
20821
|
+
limit = 100,
|
|
20822
|
+
offset = 0
|
|
20823
|
+
} = options;
|
|
20824
|
+
let query = {};
|
|
20825
|
+
if (resourceName) {
|
|
20826
|
+
query.resourceName = resourceName;
|
|
20827
|
+
}
|
|
20828
|
+
if (operation) {
|
|
20829
|
+
query.operation = operation;
|
|
20830
|
+
}
|
|
20831
|
+
if (status) {
|
|
20832
|
+
query.status = status;
|
|
20833
|
+
}
|
|
20834
|
+
const logs = await this.replicationLog.list(query);
|
|
20835
|
+
return logs.slice(offset, offset + limit);
|
|
20836
|
+
}
|
|
20837
|
+
async retryFailedReplications() {
|
|
20838
|
+
if (!this.replicationLog) {
|
|
20839
|
+
return { retried: 0 };
|
|
20840
|
+
}
|
|
20841
|
+
const failedLogs = await this.replicationLog.list({
|
|
20842
|
+
status: "failed"
|
|
20843
|
+
});
|
|
20844
|
+
let retried = 0;
|
|
20845
|
+
for (const log of failedLogs) {
|
|
20846
|
+
try {
|
|
20847
|
+
await this.queueReplication(
|
|
20848
|
+
log.resourceName,
|
|
20849
|
+
log.operation,
|
|
20850
|
+
log.recordId,
|
|
20851
|
+
log.data
|
|
20852
|
+
);
|
|
20853
|
+
retried++;
|
|
20854
|
+
} catch (error) {
|
|
20855
|
+
console.error("Failed to retry replication:", error);
|
|
20856
|
+
}
|
|
20857
|
+
}
|
|
20858
|
+
return { retried };
|
|
20859
|
+
}
|
|
20860
|
+
async syncAllData(replicatorId) {
|
|
20861
|
+
const replicator = this.replicators.find((r) => r.id === replicatorId);
|
|
20862
|
+
if (!replicator) {
|
|
20863
|
+
throw new Error(`Replicator not found: ${replicatorId}`);
|
|
20864
|
+
}
|
|
20865
|
+
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
20866
|
+
for (const resourceName in this.database.resources) {
|
|
20867
|
+
if (resourceName === "replication_logs") continue;
|
|
20868
|
+
if (replicator.instance.shouldReplicateResource(resourceName)) {
|
|
20869
|
+
this.emit("replication.sync.resource", { resourceName, replicatorId });
|
|
20870
|
+
const resource = this.database.resources[resourceName];
|
|
20871
|
+
const allRecords = await resource.getAll();
|
|
20872
|
+
for (const record of allRecords) {
|
|
20873
|
+
await replicator.instance.replicate(resourceName, "insert", record, record.id);
|
|
20874
|
+
}
|
|
20875
|
+
}
|
|
20876
|
+
}
|
|
20877
|
+
this.emit("replication.sync.completed", { replicatorId, stats: this.stats });
|
|
17825
20878
|
}
|
|
17826
20879
|
}
|
|
17827
20880
|
|
|
17828
|
-
export { AVAILABLE_BEHAVIORS, AuthenticationError, BaseError, Cache, CachePlugin, Client, ConnectionString, CostsPlugin, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, InvalidResourceItem, MemoryCache, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PermissionError, Plugin, PluginObject, Resource, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, S3Cache, S3DBError, S3_DEFAULT_ENDPOINT, S3_DEFAULT_REGION, S3db, Schema, SchemaActions, UnknownError, ValidationError, Validator, ValidatorManager, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateTotalSize, calculateUTF8Bytes, decrypt, S3db as default, encrypt, getBehavior, getSizeBreakdown, idGenerator, passwordGenerator, sha256, streamToString, transformValue };
|
|
20881
|
+
export { AVAILABLE_BEHAVIORS, AuditPlugin, AuthenticationError, BaseError, Cache, CachePlugin, Client, ConnectionString, CostsPlugin, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, FullTextPlugin, InvalidResourceItem, MemoryCache, MetricsPlugin, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PermissionError, Plugin, PluginObject, ReplicationPlugin, Resource, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, S3Cache, S3DBError, S3_DEFAULT_ENDPOINT, S3_DEFAULT_REGION, S3db, Schema, SchemaActions, UnknownError, ValidationError, Validator, ValidatorManager, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateTotalSize, calculateUTF8Bytes, decrypt, S3db as default, encrypt, getBehavior, getSizeBreakdown, idGenerator, passwordGenerator, sha256, streamToString, transformValue };
|