s3db.js 10.0.19 → 11.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +667 -212
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +667 -212
- package/dist/s3db.es.js.map +1 -1
- package/package.json +2 -1
- package/src/concerns/plugin-storage.js +443 -0
- package/src/database.class.js +48 -15
- package/src/plugins/audit.plugin.js +1 -1
- package/src/plugins/backup.plugin.js +1 -1
- package/src/plugins/cache.plugin.js +2 -6
- package/src/plugins/eventual-consistency/analytics.js +32 -8
- package/src/plugins/eventual-consistency/config.js +2 -1
- package/src/plugins/eventual-consistency/consolidation.js +73 -2
- package/src/plugins/eventual-consistency/index.js +4 -4
- package/src/plugins/eventual-consistency/{setup.js → install.js} +7 -6
- package/src/plugins/fulltext.plugin.js +3 -4
- package/src/plugins/metrics.plugin.js +10 -11
- package/src/plugins/plugin.class.js +79 -9
- package/src/plugins/queue-consumer.plugin.js +4 -3
- package/src/plugins/replicator.plugin.js +11 -13
- package/src/plugins/s3-queue.plugin.js +1 -1
- package/src/plugins/scheduler.plugin.js +8 -9
- package/src/plugins/state-machine.plugin.js +3 -4
package/dist/s3db.cjs.js
CHANGED
|
@@ -660,18 +660,496 @@ var id = /*#__PURE__*/Object.freeze({
|
|
|
660
660
|
passwordGenerator: passwordGenerator
|
|
661
661
|
});
|
|
662
662
|
|
|
663
|
+
function analyzeString(str) {
|
|
664
|
+
if (!str || typeof str !== "string") {
|
|
665
|
+
return { type: "none", safe: true };
|
|
666
|
+
}
|
|
667
|
+
let hasLatin1 = false;
|
|
668
|
+
let hasMultibyte = false;
|
|
669
|
+
let asciiCount = 0;
|
|
670
|
+
let latin1Count = 0;
|
|
671
|
+
let multibyteCount = 0;
|
|
672
|
+
for (let i = 0; i < str.length; i++) {
|
|
673
|
+
const code = str.charCodeAt(i);
|
|
674
|
+
if (code >= 32 && code <= 126) {
|
|
675
|
+
asciiCount++;
|
|
676
|
+
} else if (code < 32 || code === 127) {
|
|
677
|
+
hasMultibyte = true;
|
|
678
|
+
multibyteCount++;
|
|
679
|
+
} else if (code >= 128 && code <= 255) {
|
|
680
|
+
hasLatin1 = true;
|
|
681
|
+
latin1Count++;
|
|
682
|
+
} else {
|
|
683
|
+
hasMultibyte = true;
|
|
684
|
+
multibyteCount++;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
if (!hasLatin1 && !hasMultibyte) {
|
|
688
|
+
return {
|
|
689
|
+
type: "ascii",
|
|
690
|
+
safe: true,
|
|
691
|
+
stats: { ascii: asciiCount, latin1: 0, multibyte: 0 }
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
if (hasMultibyte) {
|
|
695
|
+
const multibyteRatio = multibyteCount / str.length;
|
|
696
|
+
if (multibyteRatio > 0.3) {
|
|
697
|
+
return {
|
|
698
|
+
type: "base64",
|
|
699
|
+
safe: false,
|
|
700
|
+
reason: "high multibyte content",
|
|
701
|
+
stats: { ascii: asciiCount, latin1: latin1Count, multibyte: multibyteCount }
|
|
702
|
+
};
|
|
703
|
+
}
|
|
704
|
+
return {
|
|
705
|
+
type: "url",
|
|
706
|
+
safe: false,
|
|
707
|
+
reason: "contains multibyte characters",
|
|
708
|
+
stats: { ascii: asciiCount, latin1: latin1Count, multibyte: multibyteCount }
|
|
709
|
+
};
|
|
710
|
+
}
|
|
711
|
+
const latin1Ratio = latin1Count / str.length;
|
|
712
|
+
if (latin1Ratio > 0.5) {
|
|
713
|
+
return {
|
|
714
|
+
type: "base64",
|
|
715
|
+
safe: false,
|
|
716
|
+
reason: "high Latin-1 content",
|
|
717
|
+
stats: { ascii: asciiCount, latin1: latin1Count, multibyte: 0 }
|
|
718
|
+
};
|
|
719
|
+
}
|
|
720
|
+
return {
|
|
721
|
+
type: "url",
|
|
722
|
+
safe: false,
|
|
723
|
+
reason: "contains Latin-1 extended characters",
|
|
724
|
+
stats: { ascii: asciiCount, latin1: latin1Count, multibyte: 0 }
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
function metadataEncode(value) {
|
|
728
|
+
if (value === null) {
|
|
729
|
+
return { encoded: "null", encoding: "special" };
|
|
730
|
+
}
|
|
731
|
+
if (value === void 0) {
|
|
732
|
+
return { encoded: "undefined", encoding: "special" };
|
|
733
|
+
}
|
|
734
|
+
const stringValue = String(value);
|
|
735
|
+
const analysis = analyzeString(stringValue);
|
|
736
|
+
switch (analysis.type) {
|
|
737
|
+
case "none":
|
|
738
|
+
case "ascii":
|
|
739
|
+
return {
|
|
740
|
+
encoded: stringValue,
|
|
741
|
+
encoding: "none",
|
|
742
|
+
analysis
|
|
743
|
+
};
|
|
744
|
+
case "url":
|
|
745
|
+
return {
|
|
746
|
+
encoded: "u:" + encodeURIComponent(stringValue),
|
|
747
|
+
encoding: "url",
|
|
748
|
+
analysis
|
|
749
|
+
};
|
|
750
|
+
case "base64":
|
|
751
|
+
return {
|
|
752
|
+
encoded: "b:" + Buffer.from(stringValue, "utf8").toString("base64"),
|
|
753
|
+
encoding: "base64",
|
|
754
|
+
analysis
|
|
755
|
+
};
|
|
756
|
+
default:
|
|
757
|
+
return {
|
|
758
|
+
encoded: "b:" + Buffer.from(stringValue, "utf8").toString("base64"),
|
|
759
|
+
encoding: "base64",
|
|
760
|
+
analysis
|
|
761
|
+
};
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
function metadataDecode(value) {
|
|
765
|
+
if (value === "null") {
|
|
766
|
+
return null;
|
|
767
|
+
}
|
|
768
|
+
if (value === "undefined") {
|
|
769
|
+
return void 0;
|
|
770
|
+
}
|
|
771
|
+
if (value === null || value === void 0 || typeof value !== "string") {
|
|
772
|
+
return value;
|
|
773
|
+
}
|
|
774
|
+
if (value.startsWith("u:")) {
|
|
775
|
+
if (value.length === 2) return value;
|
|
776
|
+
try {
|
|
777
|
+
return decodeURIComponent(value.substring(2));
|
|
778
|
+
} catch (err) {
|
|
779
|
+
return value;
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
if (value.startsWith("b:")) {
|
|
783
|
+
if (value.length === 2) return value;
|
|
784
|
+
try {
|
|
785
|
+
const decoded = Buffer.from(value.substring(2), "base64").toString("utf8");
|
|
786
|
+
return decoded;
|
|
787
|
+
} catch (err) {
|
|
788
|
+
return value;
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
if (value.length > 0 && /^[A-Za-z0-9+/]+=*$/.test(value)) {
|
|
792
|
+
try {
|
|
793
|
+
const decoded = Buffer.from(value, "base64").toString("utf8");
|
|
794
|
+
if (/[^\x00-\x7F]/.test(decoded) && Buffer.from(decoded, "utf8").toString("base64") === value) {
|
|
795
|
+
return decoded;
|
|
796
|
+
}
|
|
797
|
+
} catch {
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
return value;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
const S3_METADATA_LIMIT = 2047;
|
|
804
|
+
class PluginStorage {
|
|
805
|
+
/**
|
|
806
|
+
* @param {Object} client - S3db Client instance
|
|
807
|
+
* @param {string} pluginSlug - Plugin identifier (kebab-case)
|
|
808
|
+
*/
|
|
809
|
+
constructor(client, pluginSlug) {
|
|
810
|
+
if (!client) {
|
|
811
|
+
throw new Error("PluginStorage requires a client instance");
|
|
812
|
+
}
|
|
813
|
+
if (!pluginSlug) {
|
|
814
|
+
throw new Error("PluginStorage requires a pluginSlug");
|
|
815
|
+
}
|
|
816
|
+
this.client = client;
|
|
817
|
+
this.pluginSlug = pluginSlug;
|
|
818
|
+
}
|
|
819
|
+
/**
|
|
820
|
+
* Generate hierarchical plugin-scoped key
|
|
821
|
+
*
|
|
822
|
+
* @param {string} resourceName - Resource name (optional, for resource-scoped data)
|
|
823
|
+
* @param {...string} parts - Additional path parts
|
|
824
|
+
* @returns {string} S3 key
|
|
825
|
+
*
|
|
826
|
+
* @example
|
|
827
|
+
* // Resource-scoped: resource=wallets/plugin=eventual-consistency/balance/transactions/id=txn1
|
|
828
|
+
* getPluginKey('wallets', 'balance', 'transactions', 'id=txn1')
|
|
829
|
+
*
|
|
830
|
+
* // Global plugin data: plugin=eventual-consistency/config
|
|
831
|
+
* getPluginKey(null, 'config')
|
|
832
|
+
*/
|
|
833
|
+
getPluginKey(resourceName, ...parts) {
|
|
834
|
+
if (resourceName) {
|
|
835
|
+
return `resource=${resourceName}/plugin=${this.pluginSlug}/${parts.join("/")}`;
|
|
836
|
+
}
|
|
837
|
+
return `plugin=${this.pluginSlug}/${parts.join("/")}`;
|
|
838
|
+
}
|
|
839
|
+
/**
|
|
840
|
+
* Save data with metadata encoding and behavior support
|
|
841
|
+
*
|
|
842
|
+
* @param {string} key - S3 key
|
|
843
|
+
* @param {Object} data - Data to save
|
|
844
|
+
* @param {Object} options - Options
|
|
845
|
+
* @param {string} options.behavior - 'body-overflow' | 'body-only' | 'enforce-limits'
|
|
846
|
+
* @param {string} options.contentType - Content type (default: application/json)
|
|
847
|
+
* @returns {Promise<void>}
|
|
848
|
+
*/
|
|
849
|
+
async put(key, data, options = {}) {
|
|
850
|
+
const { behavior = "body-overflow", contentType = "application/json" } = options;
|
|
851
|
+
const { metadata, body } = this._applyBehavior(data, behavior);
|
|
852
|
+
const putParams = {
|
|
853
|
+
key,
|
|
854
|
+
metadata,
|
|
855
|
+
contentType
|
|
856
|
+
};
|
|
857
|
+
if (body !== null) {
|
|
858
|
+
putParams.body = JSON.stringify(body);
|
|
859
|
+
}
|
|
860
|
+
const [ok, err] = await tryFn(() => this.client.putObject(putParams));
|
|
861
|
+
if (!ok) {
|
|
862
|
+
throw new Error(`PluginStorage.put failed for key ${key}: ${err.message}`);
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* Get data with automatic metadata decoding
|
|
867
|
+
*
|
|
868
|
+
* @param {string} key - S3 key
|
|
869
|
+
* @returns {Promise<Object|null>} Data or null if not found
|
|
870
|
+
*/
|
|
871
|
+
async get(key) {
|
|
872
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
873
|
+
if (!ok) {
|
|
874
|
+
if (err.name === "NoSuchKey" || err.Code === "NoSuchKey") {
|
|
875
|
+
return null;
|
|
876
|
+
}
|
|
877
|
+
throw new Error(`PluginStorage.get failed for key ${key}: ${err.message}`);
|
|
878
|
+
}
|
|
879
|
+
const metadata = response.Metadata || {};
|
|
880
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
881
|
+
if (response.Body) {
|
|
882
|
+
try {
|
|
883
|
+
const bodyContent = await response.Body.transformToString();
|
|
884
|
+
if (bodyContent && bodyContent.trim()) {
|
|
885
|
+
const body = JSON.parse(bodyContent);
|
|
886
|
+
return { ...parsedMetadata, ...body };
|
|
887
|
+
}
|
|
888
|
+
} catch (parseErr) {
|
|
889
|
+
throw new Error(`PluginStorage.get failed to parse body for key ${key}: ${parseErr.message}`);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
return parsedMetadata;
|
|
893
|
+
}
|
|
894
|
+
/**
|
|
895
|
+
* Parse metadata values back to their original types
|
|
896
|
+
* @private
|
|
897
|
+
*/
|
|
898
|
+
_parseMetadataValues(metadata) {
|
|
899
|
+
const parsed = {};
|
|
900
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
901
|
+
if (typeof value === "string") {
|
|
902
|
+
if (value.startsWith("{") && value.endsWith("}") || value.startsWith("[") && value.endsWith("]")) {
|
|
903
|
+
try {
|
|
904
|
+
parsed[key] = JSON.parse(value);
|
|
905
|
+
continue;
|
|
906
|
+
} catch {
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
if (!isNaN(value) && value.trim() !== "") {
|
|
910
|
+
parsed[key] = Number(value);
|
|
911
|
+
continue;
|
|
912
|
+
}
|
|
913
|
+
if (value === "true") {
|
|
914
|
+
parsed[key] = true;
|
|
915
|
+
continue;
|
|
916
|
+
}
|
|
917
|
+
if (value === "false") {
|
|
918
|
+
parsed[key] = false;
|
|
919
|
+
continue;
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
parsed[key] = value;
|
|
923
|
+
}
|
|
924
|
+
return parsed;
|
|
925
|
+
}
|
|
926
|
+
/**
|
|
927
|
+
* List all keys with plugin prefix
|
|
928
|
+
*
|
|
929
|
+
* @param {string} prefix - Additional prefix (optional)
|
|
930
|
+
* @param {Object} options - List options
|
|
931
|
+
* @param {number} options.limit - Max number of results
|
|
932
|
+
* @returns {Promise<Array<string>>} List of keys
|
|
933
|
+
*/
|
|
934
|
+
async list(prefix = "", options = {}) {
|
|
935
|
+
const { limit } = options;
|
|
936
|
+
const fullPrefix = prefix ? `plugin=${this.pluginSlug}/${prefix}` : `plugin=${this.pluginSlug}/`;
|
|
937
|
+
const [ok, err, result] = await tryFn(
|
|
938
|
+
() => this.client.listObjects({ prefix: fullPrefix, maxKeys: limit })
|
|
939
|
+
);
|
|
940
|
+
if (!ok) {
|
|
941
|
+
throw new Error(`PluginStorage.list failed: ${err.message}`);
|
|
942
|
+
}
|
|
943
|
+
const keys = result.Contents?.map((item) => item.Key) || [];
|
|
944
|
+
return this._removeKeyPrefix(keys);
|
|
945
|
+
}
|
|
946
|
+
/**
|
|
947
|
+
* List keys for a specific resource
|
|
948
|
+
*
|
|
949
|
+
* @param {string} resourceName - Resource name
|
|
950
|
+
* @param {string} subPrefix - Additional prefix within resource (optional)
|
|
951
|
+
* @param {Object} options - List options
|
|
952
|
+
* @returns {Promise<Array<string>>} List of keys
|
|
953
|
+
*/
|
|
954
|
+
async listForResource(resourceName, subPrefix = "", options = {}) {
|
|
955
|
+
const { limit } = options;
|
|
956
|
+
const fullPrefix = subPrefix ? `resource=${resourceName}/plugin=${this.pluginSlug}/${subPrefix}` : `resource=${resourceName}/plugin=${this.pluginSlug}/`;
|
|
957
|
+
const [ok, err, result] = await tryFn(
|
|
958
|
+
() => this.client.listObjects({ prefix: fullPrefix, maxKeys: limit })
|
|
959
|
+
);
|
|
960
|
+
if (!ok) {
|
|
961
|
+
throw new Error(`PluginStorage.listForResource failed: ${err.message}`);
|
|
962
|
+
}
|
|
963
|
+
const keys = result.Contents?.map((item) => item.Key) || [];
|
|
964
|
+
return this._removeKeyPrefix(keys);
|
|
965
|
+
}
|
|
966
|
+
/**
|
|
967
|
+
* Remove client keyPrefix from keys
|
|
968
|
+
* @private
|
|
969
|
+
*/
|
|
970
|
+
_removeKeyPrefix(keys) {
|
|
971
|
+
const keyPrefix = this.client.config.keyPrefix;
|
|
972
|
+
if (!keyPrefix) return keys;
|
|
973
|
+
return keys.map((key) => key.replace(keyPrefix, "")).map((key) => key.startsWith("/") ? key.replace("/", "") : key);
|
|
974
|
+
}
|
|
975
|
+
/**
|
|
976
|
+
* Delete a single object
|
|
977
|
+
*
|
|
978
|
+
* @param {string} key - S3 key
|
|
979
|
+
* @returns {Promise<void>}
|
|
980
|
+
*/
|
|
981
|
+
async delete(key) {
|
|
982
|
+
const [ok, err] = await tryFn(() => this.client.deleteObject(key));
|
|
983
|
+
if (!ok) {
|
|
984
|
+
throw new Error(`PluginStorage.delete failed for key ${key}: ${err.message}`);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
/**
|
|
988
|
+
* Delete all plugin data (for uninstall)
|
|
989
|
+
*
|
|
990
|
+
* @param {string} resourceName - Resource name (optional, if null deletes all plugin data)
|
|
991
|
+
* @returns {Promise<number>} Number of objects deleted
|
|
992
|
+
*/
|
|
993
|
+
async deleteAll(resourceName = null) {
|
|
994
|
+
let deleted = 0;
|
|
995
|
+
if (resourceName) {
|
|
996
|
+
const keys = await this.listForResource(resourceName);
|
|
997
|
+
for (const key of keys) {
|
|
998
|
+
await this.delete(key);
|
|
999
|
+
deleted++;
|
|
1000
|
+
}
|
|
1001
|
+
} else {
|
|
1002
|
+
const allKeys = await this.client.getAllKeys({});
|
|
1003
|
+
const pluginKeys = allKeys.filter(
|
|
1004
|
+
(key) => key.includes(`plugin=${this.pluginSlug}/`)
|
|
1005
|
+
);
|
|
1006
|
+
for (const key of pluginKeys) {
|
|
1007
|
+
await this.delete(key);
|
|
1008
|
+
deleted++;
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
return deleted;
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* Batch put operations
|
|
1015
|
+
*
|
|
1016
|
+
* @param {Array<{key: string, data: Object, options?: Object}>} items - Items to save
|
|
1017
|
+
* @returns {Promise<Array<{key: string, ok: boolean, error?: Error}>>} Results
|
|
1018
|
+
*/
|
|
1019
|
+
async batchPut(items) {
|
|
1020
|
+
const results = [];
|
|
1021
|
+
for (const item of items) {
|
|
1022
|
+
const [ok, err] = await tryFn(
|
|
1023
|
+
() => this.put(item.key, item.data, item.options)
|
|
1024
|
+
);
|
|
1025
|
+
results.push({
|
|
1026
|
+
key: item.key,
|
|
1027
|
+
ok,
|
|
1028
|
+
error: err
|
|
1029
|
+
});
|
|
1030
|
+
}
|
|
1031
|
+
return results;
|
|
1032
|
+
}
|
|
1033
|
+
/**
|
|
1034
|
+
* Batch get operations
|
|
1035
|
+
*
|
|
1036
|
+
* @param {Array<string>} keys - Keys to fetch
|
|
1037
|
+
* @returns {Promise<Array<{key: string, ok: boolean, data?: Object, error?: Error}>>} Results
|
|
1038
|
+
*/
|
|
1039
|
+
async batchGet(keys) {
|
|
1040
|
+
const results = [];
|
|
1041
|
+
for (const key of keys) {
|
|
1042
|
+
const [ok, err, data] = await tryFn(() => this.get(key));
|
|
1043
|
+
results.push({
|
|
1044
|
+
key,
|
|
1045
|
+
ok,
|
|
1046
|
+
data,
|
|
1047
|
+
error: err
|
|
1048
|
+
});
|
|
1049
|
+
}
|
|
1050
|
+
return results;
|
|
1051
|
+
}
|
|
1052
|
+
/**
|
|
1053
|
+
* Apply behavior to split data between metadata and body
|
|
1054
|
+
*
|
|
1055
|
+
* @private
|
|
1056
|
+
* @param {Object} data - Data to split
|
|
1057
|
+
* @param {string} behavior - Behavior strategy
|
|
1058
|
+
* @returns {{metadata: Object, body: Object|null}}
|
|
1059
|
+
*/
|
|
1060
|
+
_applyBehavior(data, behavior) {
|
|
1061
|
+
const effectiveLimit = calculateEffectiveLimit({ s3Limit: S3_METADATA_LIMIT });
|
|
1062
|
+
let metadata = {};
|
|
1063
|
+
let body = null;
|
|
1064
|
+
switch (behavior) {
|
|
1065
|
+
case "body-overflow": {
|
|
1066
|
+
const entries = Object.entries(data);
|
|
1067
|
+
const sorted = entries.map(([key, value]) => {
|
|
1068
|
+
const jsonValue = typeof value === "object" ? JSON.stringify(value) : value;
|
|
1069
|
+
const { encoded } = metadataEncode(jsonValue);
|
|
1070
|
+
const keySize = calculateUTF8Bytes(key);
|
|
1071
|
+
const valueSize = calculateUTF8Bytes(encoded);
|
|
1072
|
+
return { key, value, jsonValue, encoded, size: keySize + valueSize };
|
|
1073
|
+
}).sort((a, b) => a.size - b.size);
|
|
1074
|
+
let currentSize = 0;
|
|
1075
|
+
for (const item of sorted) {
|
|
1076
|
+
if (currentSize + item.size <= effectiveLimit) {
|
|
1077
|
+
metadata[item.key] = item.jsonValue;
|
|
1078
|
+
currentSize += item.size;
|
|
1079
|
+
} else {
|
|
1080
|
+
if (body === null) body = {};
|
|
1081
|
+
body[item.key] = item.value;
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
break;
|
|
1085
|
+
}
|
|
1086
|
+
case "body-only": {
|
|
1087
|
+
body = data;
|
|
1088
|
+
break;
|
|
1089
|
+
}
|
|
1090
|
+
case "enforce-limits": {
|
|
1091
|
+
let currentSize = 0;
|
|
1092
|
+
for (const [key, value] of Object.entries(data)) {
|
|
1093
|
+
const jsonValue = typeof value === "object" ? JSON.stringify(value) : value;
|
|
1094
|
+
const { encoded } = metadataEncode(jsonValue);
|
|
1095
|
+
const keySize = calculateUTF8Bytes(key);
|
|
1096
|
+
const valueSize = calculateUTF8Bytes(encoded);
|
|
1097
|
+
currentSize += keySize + valueSize;
|
|
1098
|
+
if (currentSize > effectiveLimit) {
|
|
1099
|
+
throw new Error(
|
|
1100
|
+
`Data exceeds metadata limit (${currentSize} > ${effectiveLimit} bytes). Use 'body-overflow' or 'body-only' behavior.`
|
|
1101
|
+
);
|
|
1102
|
+
}
|
|
1103
|
+
metadata[key] = jsonValue;
|
|
1104
|
+
}
|
|
1105
|
+
break;
|
|
1106
|
+
}
|
|
1107
|
+
default:
|
|
1108
|
+
throw new Error(`Unknown behavior: ${behavior}. Use 'body-overflow', 'body-only', or 'enforce-limits'.`);
|
|
1109
|
+
}
|
|
1110
|
+
return { metadata, body };
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
|
|
663
1114
|
class Plugin extends EventEmitter {
|
|
664
1115
|
constructor(options = {}) {
|
|
665
1116
|
super();
|
|
666
1117
|
this.name = this.constructor.name;
|
|
667
1118
|
this.options = options;
|
|
668
1119
|
this.hooks = /* @__PURE__ */ new Map();
|
|
1120
|
+
this.slug = options.slug || this._generateSlug();
|
|
1121
|
+
this._storage = null;
|
|
669
1122
|
}
|
|
670
|
-
|
|
1123
|
+
/**
|
|
1124
|
+
* Generate kebab-case slug from class name
|
|
1125
|
+
* @private
|
|
1126
|
+
* @returns {string}
|
|
1127
|
+
*/
|
|
1128
|
+
_generateSlug() {
|
|
1129
|
+
return this.name.replace(/Plugin$/, "").replace(/([a-z])([A-Z])/g, "$1-$2").toLowerCase();
|
|
1130
|
+
}
|
|
1131
|
+
/**
|
|
1132
|
+
* Get PluginStorage instance (lazy-loaded)
|
|
1133
|
+
* @returns {PluginStorage}
|
|
1134
|
+
*/
|
|
1135
|
+
getStorage() {
|
|
1136
|
+
if (!this._storage) {
|
|
1137
|
+
if (!this.database || !this.database.client) {
|
|
1138
|
+
throw new Error("Plugin must be installed before accessing storage");
|
|
1139
|
+
}
|
|
1140
|
+
this._storage = new PluginStorage(this.database.client, this.slug);
|
|
1141
|
+
}
|
|
1142
|
+
return this._storage;
|
|
1143
|
+
}
|
|
1144
|
+
/**
|
|
1145
|
+
* Install plugin
|
|
1146
|
+
* @param {Database} database - Database instance
|
|
1147
|
+
*/
|
|
1148
|
+
async install(database) {
|
|
671
1149
|
this.database = database;
|
|
672
|
-
this.
|
|
673
|
-
await this.
|
|
674
|
-
this.
|
|
1150
|
+
this.beforeInstall();
|
|
1151
|
+
await this.onInstall();
|
|
1152
|
+
this.afterInstall();
|
|
675
1153
|
}
|
|
676
1154
|
async start() {
|
|
677
1155
|
this.beforeStart();
|
|
@@ -683,13 +1161,30 @@ class Plugin extends EventEmitter {
|
|
|
683
1161
|
await this.onStop();
|
|
684
1162
|
this.afterStop();
|
|
685
1163
|
}
|
|
1164
|
+
/**
|
|
1165
|
+
* Uninstall plugin and cleanup all data
|
|
1166
|
+
* @param {Object} options - Uninstall options
|
|
1167
|
+
* @param {boolean} options.purgeData - Delete all plugin data from S3 (default: false)
|
|
1168
|
+
*/
|
|
1169
|
+
async uninstall(options = {}) {
|
|
1170
|
+
const { purgeData = false } = options;
|
|
1171
|
+
this.beforeUninstall();
|
|
1172
|
+
await this.onUninstall(options);
|
|
1173
|
+
if (purgeData && this._storage) {
|
|
1174
|
+
const deleted = await this._storage.deleteAll();
|
|
1175
|
+
this.emit("plugin.dataPurged", { deleted });
|
|
1176
|
+
}
|
|
1177
|
+
this.afterUninstall();
|
|
1178
|
+
}
|
|
686
1179
|
// Override these methods in subclasses
|
|
687
|
-
async
|
|
1180
|
+
async onInstall() {
|
|
688
1181
|
}
|
|
689
1182
|
async onStart() {
|
|
690
1183
|
}
|
|
691
1184
|
async onStop() {
|
|
692
1185
|
}
|
|
1186
|
+
async onUninstall(options) {
|
|
1187
|
+
}
|
|
693
1188
|
// Hook management methods
|
|
694
1189
|
addHook(resource, event, handler) {
|
|
695
1190
|
if (!this.hooks.has(resource)) {
|
|
@@ -801,11 +1296,11 @@ class Plugin extends EventEmitter {
|
|
|
801
1296
|
return value ?? null;
|
|
802
1297
|
}
|
|
803
1298
|
// Event emission methods
|
|
804
|
-
|
|
805
|
-
this.emit("plugin.
|
|
1299
|
+
beforeInstall() {
|
|
1300
|
+
this.emit("plugin.beforeInstall", /* @__PURE__ */ new Date());
|
|
806
1301
|
}
|
|
807
|
-
|
|
808
|
-
this.emit("plugin.
|
|
1302
|
+
afterInstall() {
|
|
1303
|
+
this.emit("plugin.afterInstall", /* @__PURE__ */ new Date());
|
|
809
1304
|
}
|
|
810
1305
|
beforeStart() {
|
|
811
1306
|
this.emit("plugin.beforeStart", /* @__PURE__ */ new Date());
|
|
@@ -819,6 +1314,12 @@ class Plugin extends EventEmitter {
|
|
|
819
1314
|
afterStop() {
|
|
820
1315
|
this.emit("plugin.afterStop", /* @__PURE__ */ new Date());
|
|
821
1316
|
}
|
|
1317
|
+
beforeUninstall() {
|
|
1318
|
+
this.emit("plugin.beforeUninstall", /* @__PURE__ */ new Date());
|
|
1319
|
+
}
|
|
1320
|
+
afterUninstall() {
|
|
1321
|
+
this.emit("plugin.afterUninstall", /* @__PURE__ */ new Date());
|
|
1322
|
+
}
|
|
822
1323
|
}
|
|
823
1324
|
|
|
824
1325
|
const PluginObject = {
|
|
@@ -841,7 +1342,7 @@ class AuditPlugin extends Plugin {
|
|
|
841
1342
|
...options
|
|
842
1343
|
};
|
|
843
1344
|
}
|
|
844
|
-
async
|
|
1345
|
+
async onInstall() {
|
|
845
1346
|
const [ok, err, auditResource] = await tryFn(() => this.database.createResource({
|
|
846
1347
|
name: "plg_audits",
|
|
847
1348
|
attributes: {
|
|
@@ -1932,7 +2433,7 @@ class BackupPlugin extends Plugin {
|
|
|
1932
2433
|
throw new Error("BackupPlugin: Invalid compression type. Use: none, gzip, brotli, deflate");
|
|
1933
2434
|
}
|
|
1934
2435
|
}
|
|
1935
|
-
async
|
|
2436
|
+
async onInstall() {
|
|
1936
2437
|
this.driver = createBackupDriver(this.config.driver, this.config.driverConfig);
|
|
1937
2438
|
await this.driver.setup(this.database);
|
|
1938
2439
|
await promises.mkdir(this.config.tempDir, { recursive: true });
|
|
@@ -3817,10 +4318,7 @@ class CachePlugin extends Plugin {
|
|
|
3817
4318
|
verbose: options.verbose || false
|
|
3818
4319
|
};
|
|
3819
4320
|
}
|
|
3820
|
-
async
|
|
3821
|
-
await super.setup(database);
|
|
3822
|
-
}
|
|
3823
|
-
async onSetup() {
|
|
4321
|
+
async onInstall() {
|
|
3824
4322
|
if (this.config.driver && typeof this.config.driver === "object") {
|
|
3825
4323
|
this.driver = this.config.driver;
|
|
3826
4324
|
} else if (this.config.driver === "memory") {
|
|
@@ -4366,7 +4864,9 @@ function createConfig(options, detectedTimezone) {
|
|
|
4366
4864
|
deleteConsolidatedTransactions: checkpoints.deleteConsolidated !== false,
|
|
4367
4865
|
autoCheckpoint: checkpoints.auto !== false,
|
|
4368
4866
|
// Debug
|
|
4369
|
-
verbose: options.verbose
|
|
4867
|
+
verbose: options.verbose !== false,
|
|
4868
|
+
// Default: true (can disable with verbose: false)
|
|
4869
|
+
debug: options.debug || false
|
|
4370
4870
|
};
|
|
4371
4871
|
}
|
|
4372
4872
|
function validateResourcesConfig(resources) {
|
|
@@ -4905,11 +5405,55 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
4905
5405
|
`[EventualConsistency] ${config.resource}.${config.field} - ${originalId}: ${currentValue} \u2192 ${consolidatedValue} (${consolidatedValue > currentValue ? "+" : ""}${consolidatedValue - currentValue})`
|
|
4906
5406
|
);
|
|
4907
5407
|
}
|
|
4908
|
-
|
|
5408
|
+
if (config.debug || config.verbose) {
|
|
5409
|
+
console.log(
|
|
5410
|
+
`\u{1F525} [DEBUG] BEFORE targetResource.update() {
|
|
5411
|
+
originalId: '${originalId}',
|
|
5412
|
+
field: '${config.field}',
|
|
5413
|
+
consolidatedValue: ${consolidatedValue},
|
|
5414
|
+
currentValue: ${currentValue}
|
|
5415
|
+
}`
|
|
5416
|
+
);
|
|
5417
|
+
}
|
|
5418
|
+
const [updateOk, updateErr, updateResult] = await tryFn(
|
|
4909
5419
|
() => targetResource.update(originalId, {
|
|
4910
5420
|
[config.field]: consolidatedValue
|
|
4911
5421
|
})
|
|
4912
5422
|
);
|
|
5423
|
+
if (config.debug || config.verbose) {
|
|
5424
|
+
console.log(
|
|
5425
|
+
`\u{1F525} [DEBUG] AFTER targetResource.update() {
|
|
5426
|
+
updateOk: ${updateOk},
|
|
5427
|
+
updateErr: ${updateErr?.message || "undefined"},
|
|
5428
|
+
updateResult: ${JSON.stringify(updateResult, null, 2)},
|
|
5429
|
+
hasField: ${updateResult?.[config.field]}
|
|
5430
|
+
}`
|
|
5431
|
+
);
|
|
5432
|
+
}
|
|
5433
|
+
if (updateOk && (config.debug || config.verbose)) {
|
|
5434
|
+
const [verifyOk, verifyErr, verifiedRecord] = await tryFn(
|
|
5435
|
+
() => targetResource.get(originalId, { skipCache: true })
|
|
5436
|
+
);
|
|
5437
|
+
console.log(
|
|
5438
|
+
`\u{1F525} [DEBUG] VERIFICATION (fresh from S3, no cache) {
|
|
5439
|
+
verifyOk: ${verifyOk},
|
|
5440
|
+
verifiedRecord[${config.field}]: ${verifiedRecord?.[config.field]},
|
|
5441
|
+
expectedValue: ${consolidatedValue},
|
|
5442
|
+
\u2705 MATCH: ${verifiedRecord?.[config.field] === consolidatedValue}
|
|
5443
|
+
}`
|
|
5444
|
+
);
|
|
5445
|
+
if (verifyOk && verifiedRecord?.[config.field] !== consolidatedValue) {
|
|
5446
|
+
console.error(
|
|
5447
|
+
`\u274C [CRITICAL BUG] Update reported success but value not persisted!
|
|
5448
|
+
Resource: ${config.resource}
|
|
5449
|
+
Field: ${config.field}
|
|
5450
|
+
Record ID: ${originalId}
|
|
5451
|
+
Expected: ${consolidatedValue}
|
|
5452
|
+
Actually got: ${verifiedRecord?.[config.field]}
|
|
5453
|
+
This indicates a bug in s3db.js resource.update()`
|
|
5454
|
+
);
|
|
5455
|
+
}
|
|
5456
|
+
}
|
|
4913
5457
|
if (!updateOk) {
|
|
4914
5458
|
if (updateErr?.message?.includes("does not exist")) {
|
|
4915
5459
|
if (config.verbose) {
|
|
@@ -4940,7 +5484,20 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
4940
5484
|
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
4941
5485
|
}
|
|
4942
5486
|
if (config.enableAnalytics && transactionsToUpdate.length > 0 && updateAnalyticsFn) {
|
|
4943
|
-
await
|
|
5487
|
+
const [analyticsOk, analyticsErr] = await tryFn(
|
|
5488
|
+
() => updateAnalyticsFn(transactionsToUpdate)
|
|
5489
|
+
);
|
|
5490
|
+
if (!analyticsOk) {
|
|
5491
|
+
console.error(
|
|
5492
|
+
`[EventualConsistency] ${config.resource}.${config.field} - CRITICAL: Analytics update failed for ${originalId}, but consolidation succeeded:`,
|
|
5493
|
+
{
|
|
5494
|
+
error: analyticsErr?.message || analyticsErr,
|
|
5495
|
+
stack: analyticsErr?.stack,
|
|
5496
|
+
originalId,
|
|
5497
|
+
transactionCount: transactionsToUpdate.length
|
|
5498
|
+
}
|
|
5499
|
+
);
|
|
5500
|
+
}
|
|
4944
5501
|
}
|
|
4945
5502
|
if (targetResource && targetResource.cache && typeof targetResource.cache.delete === "function") {
|
|
4946
5503
|
try {
|
|
@@ -5203,7 +5760,16 @@ async function runGarbageCollection(transactionResource, lockResource, config, e
|
|
|
5203
5760
|
|
|
5204
5761
|
async function updateAnalytics(transactions, analyticsResource, config) {
|
|
5205
5762
|
if (!analyticsResource || transactions.length === 0) return;
|
|
5206
|
-
if (config.
|
|
5763
|
+
if (!config.field) {
|
|
5764
|
+
throw new Error(
|
|
5765
|
+
`[EventualConsistency] CRITICAL BUG: config.field is undefined in updateAnalytics()!
|
|
5766
|
+
This indicates a race condition in the plugin where multiple handlers are sharing the same config object.
|
|
5767
|
+
Config: ${JSON.stringify({ resource: config.resource, field: config.field, verbose: config.verbose })}
|
|
5768
|
+
Transactions count: ${transactions.length}
|
|
5769
|
+
AnalyticsResource: ${analyticsResource?.name || "unknown"}`
|
|
5770
|
+
);
|
|
5771
|
+
}
|
|
5772
|
+
if (config.verbose || config.debug) {
|
|
5207
5773
|
console.log(
|
|
5208
5774
|
`[EventualConsistency] ${config.resource}.${config.field} - Updating analytics for ${transactions.length} transactions...`
|
|
5209
5775
|
);
|
|
@@ -5211,7 +5777,7 @@ async function updateAnalytics(transactions, analyticsResource, config) {
|
|
|
5211
5777
|
try {
|
|
5212
5778
|
const byHour = groupByCohort(transactions, "cohortHour");
|
|
5213
5779
|
const cohortCount = Object.keys(byHour).length;
|
|
5214
|
-
if (config.verbose) {
|
|
5780
|
+
if (config.verbose || config.debug) {
|
|
5215
5781
|
console.log(
|
|
5216
5782
|
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
5217
5783
|
);
|
|
@@ -5221,7 +5787,7 @@ async function updateAnalytics(transactions, analyticsResource, config) {
|
|
|
5221
5787
|
}
|
|
5222
5788
|
if (config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5223
5789
|
const uniqueHours = Object.keys(byHour);
|
|
5224
|
-
if (config.verbose) {
|
|
5790
|
+
if (config.verbose || config.debug) {
|
|
5225
5791
|
console.log(
|
|
5226
5792
|
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
5227
5793
|
);
|
|
@@ -5230,15 +5796,24 @@ async function updateAnalytics(transactions, analyticsResource, config) {
|
|
|
5230
5796
|
await rollupAnalytics(cohortHour, analyticsResource, config);
|
|
5231
5797
|
}
|
|
5232
5798
|
}
|
|
5233
|
-
if (config.verbose) {
|
|
5799
|
+
if (config.verbose || config.debug) {
|
|
5234
5800
|
console.log(
|
|
5235
5801
|
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update complete for ${cohortCount} cohorts`
|
|
5236
5802
|
);
|
|
5237
5803
|
}
|
|
5238
5804
|
} catch (error) {
|
|
5239
|
-
console.
|
|
5240
|
-
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update
|
|
5241
|
-
|
|
5805
|
+
console.error(
|
|
5806
|
+
`[EventualConsistency] CRITICAL: ${config.resource}.${config.field} - Analytics update failed:`,
|
|
5807
|
+
{
|
|
5808
|
+
error: error.message,
|
|
5809
|
+
stack: error.stack,
|
|
5810
|
+
field: config.field,
|
|
5811
|
+
resource: config.resource,
|
|
5812
|
+
transactionCount: transactions.length
|
|
5813
|
+
}
|
|
5814
|
+
);
|
|
5815
|
+
throw new Error(
|
|
5816
|
+
`Analytics update failed for ${config.resource}.${config.field}: ${error.message}`
|
|
5242
5817
|
);
|
|
5243
5818
|
}
|
|
5244
5819
|
}
|
|
@@ -5290,6 +5865,7 @@ async function upsertAnalytics(period, cohort, transactions, analyticsResource,
|
|
|
5290
5865
|
await tryFn(
|
|
5291
5866
|
() => analyticsResource.insert({
|
|
5292
5867
|
id,
|
|
5868
|
+
field: config.field,
|
|
5293
5869
|
period,
|
|
5294
5870
|
cohort,
|
|
5295
5871
|
transactionCount,
|
|
@@ -5321,8 +5897,8 @@ function calculateOperationBreakdown(transactions) {
|
|
|
5321
5897
|
async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
5322
5898
|
const cohortDate = cohortHour.substring(0, 10);
|
|
5323
5899
|
const cohortMonth = cohortHour.substring(0, 7);
|
|
5324
|
-
await rollupPeriod("day", cohortDate, cohortDate, analyticsResource);
|
|
5325
|
-
await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource);
|
|
5900
|
+
await rollupPeriod("day", cohortDate, cohortDate, analyticsResource, config);
|
|
5901
|
+
await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource, config);
|
|
5326
5902
|
}
|
|
5327
5903
|
async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
|
|
5328
5904
|
const sourcePeriod = period === "day" ? "hour" : "day";
|
|
@@ -5372,6 +5948,7 @@ async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, con
|
|
|
5372
5948
|
await tryFn(
|
|
5373
5949
|
() => analyticsResource.insert({
|
|
5374
5950
|
id,
|
|
5951
|
+
field: config.field,
|
|
5375
5952
|
period,
|
|
5376
5953
|
cohort,
|
|
5377
5954
|
transactionCount,
|
|
@@ -5731,7 +6308,7 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
5731
6308
|
};
|
|
5732
6309
|
}
|
|
5733
6310
|
|
|
5734
|
-
async function
|
|
6311
|
+
async function onInstall(database, fieldHandlers, completeFieldSetupFn, watchForResourceFn) {
|
|
5735
6312
|
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5736
6313
|
const targetResource = database.resources[resourceName];
|
|
5737
6314
|
if (!targetResource) {
|
|
@@ -5831,6 +6408,7 @@ async function createAnalyticsResource(handler, database, resourceName, fieldNam
|
|
|
5831
6408
|
name: analyticsResourceName,
|
|
5832
6409
|
attributes: {
|
|
5833
6410
|
id: "string|required",
|
|
6411
|
+
field: "string|required",
|
|
5834
6412
|
period: "string|required",
|
|
5835
6413
|
cohort: "string|required",
|
|
5836
6414
|
transactionCount: "number|required",
|
|
@@ -5928,10 +6506,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5928
6506
|
logInitialization(this.config, this.fieldHandlers, timezoneAutoDetected);
|
|
5929
6507
|
}
|
|
5930
6508
|
/**
|
|
5931
|
-
*
|
|
6509
|
+
* Install hook - create resources and register helpers
|
|
5932
6510
|
*/
|
|
5933
|
-
async
|
|
5934
|
-
await
|
|
6511
|
+
async onInstall() {
|
|
6512
|
+
await onInstall(
|
|
5935
6513
|
this.database,
|
|
5936
6514
|
this.fieldHandlers,
|
|
5937
6515
|
(handler) => completeFieldSetup(handler, this.database, this.config, this),
|
|
@@ -6296,9 +6874,8 @@ class FullTextPlugin extends Plugin {
|
|
|
6296
6874
|
};
|
|
6297
6875
|
this.indexes = /* @__PURE__ */ new Map();
|
|
6298
6876
|
}
|
|
6299
|
-
async
|
|
6300
|
-
|
|
6301
|
-
const [ok, err, indexResource] = await tryFn(() => database.createResource({
|
|
6877
|
+
async onInstall() {
|
|
6878
|
+
const [ok, err, indexResource] = await tryFn(() => this.database.createResource({
|
|
6302
6879
|
name: "plg_fulltext_indexes",
|
|
6303
6880
|
attributes: {
|
|
6304
6881
|
id: "string|required",
|
|
@@ -6311,7 +6888,7 @@ class FullTextPlugin extends Plugin {
|
|
|
6311
6888
|
lastUpdated: "string|required"
|
|
6312
6889
|
}
|
|
6313
6890
|
}));
|
|
6314
|
-
this.indexResource = ok ? indexResource : database.resources.fulltext_indexes;
|
|
6891
|
+
this.indexResource = ok ? indexResource : this.database.resources.fulltext_indexes;
|
|
6315
6892
|
await this.loadIndexes();
|
|
6316
6893
|
this.installDatabaseHooks();
|
|
6317
6894
|
this.installIndexingHooks();
|
|
@@ -6677,11 +7254,10 @@ class MetricsPlugin extends Plugin {
|
|
|
6677
7254
|
};
|
|
6678
7255
|
this.flushTimer = null;
|
|
6679
7256
|
}
|
|
6680
|
-
async
|
|
6681
|
-
this.database = database;
|
|
7257
|
+
async onInstall() {
|
|
6682
7258
|
if (typeof process !== "undefined" && process.env.NODE_ENV === "test") return;
|
|
6683
7259
|
const [ok, err] = await tryFn(async () => {
|
|
6684
|
-
const [ok1, err1, metricsResource] = await tryFn(() => database.createResource({
|
|
7260
|
+
const [ok1, err1, metricsResource] = await tryFn(() => this.database.createResource({
|
|
6685
7261
|
name: "plg_metrics",
|
|
6686
7262
|
attributes: {
|
|
6687
7263
|
id: "string|required",
|
|
@@ -6697,8 +7273,8 @@ class MetricsPlugin extends Plugin {
|
|
|
6697
7273
|
metadata: "json"
|
|
6698
7274
|
}
|
|
6699
7275
|
}));
|
|
6700
|
-
this.metricsResource = ok1 ? metricsResource : database.resources.plg_metrics;
|
|
6701
|
-
const [ok2, err2, errorsResource] = await tryFn(() => database.createResource({
|
|
7276
|
+
this.metricsResource = ok1 ? metricsResource : this.database.resources.plg_metrics;
|
|
7277
|
+
const [ok2, err2, errorsResource] = await tryFn(() => this.database.createResource({
|
|
6702
7278
|
name: "plg_error_logs",
|
|
6703
7279
|
attributes: {
|
|
6704
7280
|
id: "string|required",
|
|
@@ -6709,8 +7285,8 @@ class MetricsPlugin extends Plugin {
|
|
|
6709
7285
|
metadata: "json"
|
|
6710
7286
|
}
|
|
6711
7287
|
}));
|
|
6712
|
-
this.errorsResource = ok2 ? errorsResource : database.resources.plg_error_logs;
|
|
6713
|
-
const [ok3, err3, performanceResource] = await tryFn(() => database.createResource({
|
|
7288
|
+
this.errorsResource = ok2 ? errorsResource : this.database.resources.plg_error_logs;
|
|
7289
|
+
const [ok3, err3, performanceResource] = await tryFn(() => this.database.createResource({
|
|
6714
7290
|
name: "plg_performance_logs",
|
|
6715
7291
|
attributes: {
|
|
6716
7292
|
id: "string|required",
|
|
@@ -6721,12 +7297,12 @@ class MetricsPlugin extends Plugin {
|
|
|
6721
7297
|
metadata: "json"
|
|
6722
7298
|
}
|
|
6723
7299
|
}));
|
|
6724
|
-
this.performanceResource = ok3 ? performanceResource : database.resources.plg_performance_logs;
|
|
7300
|
+
this.performanceResource = ok3 ? performanceResource : this.database.resources.plg_performance_logs;
|
|
6725
7301
|
});
|
|
6726
7302
|
if (!ok) {
|
|
6727
|
-
this.metricsResource = database.resources.plg_metrics;
|
|
6728
|
-
this.errorsResource = database.resources.plg_error_logs;
|
|
6729
|
-
this.performanceResource = database.resources.plg_performance_logs;
|
|
7303
|
+
this.metricsResource = this.database.resources.plg_metrics;
|
|
7304
|
+
this.errorsResource = this.database.resources.plg_error_logs;
|
|
7305
|
+
this.performanceResource = this.database.resources.plg_performance_logs;
|
|
6730
7306
|
}
|
|
6731
7307
|
this.installDatabaseHooks();
|
|
6732
7308
|
this.installMetricsHooks();
|
|
@@ -7304,14 +7880,14 @@ function createConsumer(driver, config) {
|
|
|
7304
7880
|
return new ConsumerClass(config);
|
|
7305
7881
|
}
|
|
7306
7882
|
|
|
7307
|
-
class QueueConsumerPlugin {
|
|
7883
|
+
class QueueConsumerPlugin extends Plugin {
|
|
7308
7884
|
constructor(options = {}) {
|
|
7885
|
+
super(options);
|
|
7309
7886
|
this.options = options;
|
|
7310
7887
|
this.driversConfig = Array.isArray(options.consumers) ? options.consumers : [];
|
|
7311
7888
|
this.consumers = [];
|
|
7312
7889
|
}
|
|
7313
|
-
async
|
|
7314
|
-
this.database = database;
|
|
7890
|
+
async onInstall() {
|
|
7315
7891
|
for (const driverDef of this.driversConfig) {
|
|
7316
7892
|
const { driver, config: driverConfig = {}, consumers: consumerDefs = [] } = driverDef;
|
|
7317
7893
|
if (consumerDefs.length === 0 && driverDef.resources) {
|
|
@@ -8096,146 +8672,6 @@ class PostgresReplicator extends BaseReplicator {
|
|
|
8096
8672
|
}
|
|
8097
8673
|
}
|
|
8098
8674
|
|
|
8099
|
-
function analyzeString(str) {
|
|
8100
|
-
if (!str || typeof str !== "string") {
|
|
8101
|
-
return { type: "none", safe: true };
|
|
8102
|
-
}
|
|
8103
|
-
let hasLatin1 = false;
|
|
8104
|
-
let hasMultibyte = false;
|
|
8105
|
-
let asciiCount = 0;
|
|
8106
|
-
let latin1Count = 0;
|
|
8107
|
-
let multibyteCount = 0;
|
|
8108
|
-
for (let i = 0; i < str.length; i++) {
|
|
8109
|
-
const code = str.charCodeAt(i);
|
|
8110
|
-
if (code >= 32 && code <= 126) {
|
|
8111
|
-
asciiCount++;
|
|
8112
|
-
} else if (code < 32 || code === 127) {
|
|
8113
|
-
hasMultibyte = true;
|
|
8114
|
-
multibyteCount++;
|
|
8115
|
-
} else if (code >= 128 && code <= 255) {
|
|
8116
|
-
hasLatin1 = true;
|
|
8117
|
-
latin1Count++;
|
|
8118
|
-
} else {
|
|
8119
|
-
hasMultibyte = true;
|
|
8120
|
-
multibyteCount++;
|
|
8121
|
-
}
|
|
8122
|
-
}
|
|
8123
|
-
if (!hasLatin1 && !hasMultibyte) {
|
|
8124
|
-
return {
|
|
8125
|
-
type: "ascii",
|
|
8126
|
-
safe: true,
|
|
8127
|
-
stats: { ascii: asciiCount, latin1: 0, multibyte: 0 }
|
|
8128
|
-
};
|
|
8129
|
-
}
|
|
8130
|
-
if (hasMultibyte) {
|
|
8131
|
-
const multibyteRatio = multibyteCount / str.length;
|
|
8132
|
-
if (multibyteRatio > 0.3) {
|
|
8133
|
-
return {
|
|
8134
|
-
type: "base64",
|
|
8135
|
-
safe: false,
|
|
8136
|
-
reason: "high multibyte content",
|
|
8137
|
-
stats: { ascii: asciiCount, latin1: latin1Count, multibyte: multibyteCount }
|
|
8138
|
-
};
|
|
8139
|
-
}
|
|
8140
|
-
return {
|
|
8141
|
-
type: "url",
|
|
8142
|
-
safe: false,
|
|
8143
|
-
reason: "contains multibyte characters",
|
|
8144
|
-
stats: { ascii: asciiCount, latin1: latin1Count, multibyte: multibyteCount }
|
|
8145
|
-
};
|
|
8146
|
-
}
|
|
8147
|
-
const latin1Ratio = latin1Count / str.length;
|
|
8148
|
-
if (latin1Ratio > 0.5) {
|
|
8149
|
-
return {
|
|
8150
|
-
type: "base64",
|
|
8151
|
-
safe: false,
|
|
8152
|
-
reason: "high Latin-1 content",
|
|
8153
|
-
stats: { ascii: asciiCount, latin1: latin1Count, multibyte: 0 }
|
|
8154
|
-
};
|
|
8155
|
-
}
|
|
8156
|
-
return {
|
|
8157
|
-
type: "url",
|
|
8158
|
-
safe: false,
|
|
8159
|
-
reason: "contains Latin-1 extended characters",
|
|
8160
|
-
stats: { ascii: asciiCount, latin1: latin1Count, multibyte: 0 }
|
|
8161
|
-
};
|
|
8162
|
-
}
|
|
8163
|
-
function metadataEncode(value) {
|
|
8164
|
-
if (value === null) {
|
|
8165
|
-
return { encoded: "null", encoding: "special" };
|
|
8166
|
-
}
|
|
8167
|
-
if (value === void 0) {
|
|
8168
|
-
return { encoded: "undefined", encoding: "special" };
|
|
8169
|
-
}
|
|
8170
|
-
const stringValue = String(value);
|
|
8171
|
-
const analysis = analyzeString(stringValue);
|
|
8172
|
-
switch (analysis.type) {
|
|
8173
|
-
case "none":
|
|
8174
|
-
case "ascii":
|
|
8175
|
-
return {
|
|
8176
|
-
encoded: stringValue,
|
|
8177
|
-
encoding: "none",
|
|
8178
|
-
analysis
|
|
8179
|
-
};
|
|
8180
|
-
case "url":
|
|
8181
|
-
return {
|
|
8182
|
-
encoded: "u:" + encodeURIComponent(stringValue),
|
|
8183
|
-
encoding: "url",
|
|
8184
|
-
analysis
|
|
8185
|
-
};
|
|
8186
|
-
case "base64":
|
|
8187
|
-
return {
|
|
8188
|
-
encoded: "b:" + Buffer.from(stringValue, "utf8").toString("base64"),
|
|
8189
|
-
encoding: "base64",
|
|
8190
|
-
analysis
|
|
8191
|
-
};
|
|
8192
|
-
default:
|
|
8193
|
-
return {
|
|
8194
|
-
encoded: "b:" + Buffer.from(stringValue, "utf8").toString("base64"),
|
|
8195
|
-
encoding: "base64",
|
|
8196
|
-
analysis
|
|
8197
|
-
};
|
|
8198
|
-
}
|
|
8199
|
-
}
|
|
8200
|
-
function metadataDecode(value) {
|
|
8201
|
-
if (value === "null") {
|
|
8202
|
-
return null;
|
|
8203
|
-
}
|
|
8204
|
-
if (value === "undefined") {
|
|
8205
|
-
return void 0;
|
|
8206
|
-
}
|
|
8207
|
-
if (value === null || value === void 0 || typeof value !== "string") {
|
|
8208
|
-
return value;
|
|
8209
|
-
}
|
|
8210
|
-
if (value.startsWith("u:")) {
|
|
8211
|
-
if (value.length === 2) return value;
|
|
8212
|
-
try {
|
|
8213
|
-
return decodeURIComponent(value.substring(2));
|
|
8214
|
-
} catch (err) {
|
|
8215
|
-
return value;
|
|
8216
|
-
}
|
|
8217
|
-
}
|
|
8218
|
-
if (value.startsWith("b:")) {
|
|
8219
|
-
if (value.length === 2) return value;
|
|
8220
|
-
try {
|
|
8221
|
-
const decoded = Buffer.from(value.substring(2), "base64").toString("utf8");
|
|
8222
|
-
return decoded;
|
|
8223
|
-
} catch (err) {
|
|
8224
|
-
return value;
|
|
8225
|
-
}
|
|
8226
|
-
}
|
|
8227
|
-
if (value.length > 0 && /^[A-Za-z0-9+/]+=*$/.test(value)) {
|
|
8228
|
-
try {
|
|
8229
|
-
const decoded = Buffer.from(value, "base64").toString("utf8");
|
|
8230
|
-
if (/[^\x00-\x7F]/.test(decoded) && Buffer.from(decoded, "utf8").toString("base64") === value) {
|
|
8231
|
-
return decoded;
|
|
8232
|
-
}
|
|
8233
|
-
} catch {
|
|
8234
|
-
}
|
|
8235
|
-
}
|
|
8236
|
-
return value;
|
|
8237
|
-
}
|
|
8238
|
-
|
|
8239
8675
|
const S3_DEFAULT_REGION = "us-east-1";
|
|
8240
8676
|
const S3_DEFAULT_ENDPOINT = "https://s3.us-east-1.amazonaws.com";
|
|
8241
8677
|
class ConnectionString {
|
|
@@ -12352,7 +12788,7 @@ class Database extends EventEmitter {
|
|
|
12352
12788
|
this.id = idGenerator(7);
|
|
12353
12789
|
this.version = "1";
|
|
12354
12790
|
this.s3dbVersion = (() => {
|
|
12355
|
-
const [ok, err, version] = tryFn(() => true ? "
|
|
12791
|
+
const [ok, err, version] = tryFn(() => true ? "11.0.1" : "latest");
|
|
12356
12792
|
return ok ? version : "latest";
|
|
12357
12793
|
})();
|
|
12358
12794
|
this.resources = {};
|
|
@@ -12652,18 +13088,14 @@ class Database extends EventEmitter {
|
|
|
12652
13088
|
const db = this;
|
|
12653
13089
|
if (!lodashEs.isEmpty(this.pluginList)) {
|
|
12654
13090
|
const plugins = this.pluginList.map((p) => lodashEs.isFunction(p) ? new p(this) : p);
|
|
12655
|
-
const
|
|
12656
|
-
|
|
12657
|
-
await plugin.setup(db);
|
|
12658
|
-
if (plugin.afterSetup) await plugin.afterSetup();
|
|
13091
|
+
const installProms = plugins.map(async (plugin) => {
|
|
13092
|
+
await plugin.install(db);
|
|
12659
13093
|
const pluginName = this._getPluginName(plugin);
|
|
12660
13094
|
this.pluginRegistry[pluginName] = plugin;
|
|
12661
13095
|
});
|
|
12662
|
-
await Promise.all(
|
|
13096
|
+
await Promise.all(installProms);
|
|
12663
13097
|
const startProms = plugins.map(async (plugin) => {
|
|
12664
|
-
if (plugin.beforeStart) await plugin.beforeStart();
|
|
12665
13098
|
await plugin.start();
|
|
12666
|
-
if (plugin.afterStart) await plugin.afterStart();
|
|
12667
13099
|
});
|
|
12668
13100
|
await Promise.all(startProms);
|
|
12669
13101
|
}
|
|
@@ -12684,11 +13116,37 @@ class Database extends EventEmitter {
|
|
|
12684
13116
|
const pluginName = this._getPluginName(plugin, name);
|
|
12685
13117
|
this.plugins[pluginName] = plugin;
|
|
12686
13118
|
if (this.isConnected()) {
|
|
12687
|
-
await plugin.
|
|
13119
|
+
await plugin.install(this);
|
|
12688
13120
|
await plugin.start();
|
|
12689
13121
|
}
|
|
12690
13122
|
return plugin;
|
|
12691
13123
|
}
|
|
13124
|
+
/**
|
|
13125
|
+
* Uninstall a plugin and optionally purge its data
|
|
13126
|
+
* @param {string} name - Plugin name
|
|
13127
|
+
* @param {Object} options - Uninstall options
|
|
13128
|
+
* @param {boolean} options.purgeData - Delete all plugin data from S3 (default: false)
|
|
13129
|
+
*/
|
|
13130
|
+
async uninstallPlugin(name, options = {}) {
|
|
13131
|
+
const pluginName = name.toLowerCase().replace("plugin", "");
|
|
13132
|
+
const plugin = this.plugins[pluginName] || this.pluginRegistry[pluginName];
|
|
13133
|
+
if (!plugin) {
|
|
13134
|
+
throw new Error(`Plugin '${name}' not found`);
|
|
13135
|
+
}
|
|
13136
|
+
if (plugin.stop) {
|
|
13137
|
+
await plugin.stop();
|
|
13138
|
+
}
|
|
13139
|
+
if (plugin.uninstall) {
|
|
13140
|
+
await plugin.uninstall(options);
|
|
13141
|
+
}
|
|
13142
|
+
delete this.plugins[pluginName];
|
|
13143
|
+
delete this.pluginRegistry[pluginName];
|
|
13144
|
+
const index = this.pluginList.indexOf(plugin);
|
|
13145
|
+
if (index > -1) {
|
|
13146
|
+
this.pluginList.splice(index, 1);
|
|
13147
|
+
}
|
|
13148
|
+
this.emit("plugin.uninstalled", { name: pluginName, plugin });
|
|
13149
|
+
}
|
|
12692
13150
|
async uploadMetadataFile() {
|
|
12693
13151
|
const metadata = {
|
|
12694
13152
|
version: this.version,
|
|
@@ -14155,10 +14613,9 @@ class ReplicatorPlugin extends Plugin {
|
|
|
14155
14613
|
resource.on("delete", deleteHandler);
|
|
14156
14614
|
this.eventListenersInstalled.add(resource.name);
|
|
14157
14615
|
}
|
|
14158
|
-
async
|
|
14159
|
-
this.database = database;
|
|
14616
|
+
async onInstall() {
|
|
14160
14617
|
if (this.config.persistReplicatorLog) {
|
|
14161
|
-
const [ok, err, logResource] = await tryFn(() => database.createResource({
|
|
14618
|
+
const [ok, err, logResource] = await tryFn(() => this.database.createResource({
|
|
14162
14619
|
name: this.config.replicatorLogResource || "plg_replicator_logs",
|
|
14163
14620
|
attributes: {
|
|
14164
14621
|
id: "string|required",
|
|
@@ -14173,14 +14630,14 @@ class ReplicatorPlugin extends Plugin {
|
|
|
14173
14630
|
if (ok) {
|
|
14174
14631
|
this.replicatorLogResource = logResource;
|
|
14175
14632
|
} else {
|
|
14176
|
-
this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "plg_replicator_logs"];
|
|
14633
|
+
this.replicatorLogResource = this.database.resources[this.config.replicatorLogResource || "plg_replicator_logs"];
|
|
14177
14634
|
}
|
|
14178
14635
|
}
|
|
14179
|
-
await this.initializeReplicators(database);
|
|
14636
|
+
await this.initializeReplicators(this.database);
|
|
14180
14637
|
this.installDatabaseHooks();
|
|
14181
|
-
for (const resource of Object.values(database.resources)) {
|
|
14638
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
14182
14639
|
if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
|
|
14183
|
-
this.installEventListeners(resource, database, this);
|
|
14640
|
+
this.installEventListeners(resource, this.database, this);
|
|
14184
14641
|
}
|
|
14185
14642
|
}
|
|
14186
14643
|
}
|
|
@@ -14224,8 +14681,8 @@ class ReplicatorPlugin extends Plugin {
|
|
|
14224
14681
|
}
|
|
14225
14682
|
}
|
|
14226
14683
|
async uploadMetadataFile(database) {
|
|
14227
|
-
if (typeof database.uploadMetadataFile === "function") {
|
|
14228
|
-
await database.uploadMetadataFile();
|
|
14684
|
+
if (typeof this.database.uploadMetadataFile === "function") {
|
|
14685
|
+
await this.database.uploadMetadataFile();
|
|
14229
14686
|
}
|
|
14230
14687
|
}
|
|
14231
14688
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
@@ -14599,7 +15056,7 @@ class S3QueuePlugin extends Plugin {
|
|
|
14599
15056
|
this.cacheCleanupInterval = null;
|
|
14600
15057
|
this.lockCleanupInterval = null;
|
|
14601
15058
|
}
|
|
14602
|
-
async
|
|
15059
|
+
async onInstall() {
|
|
14603
15060
|
this.targetResource = this.database.resources[this.config.resource];
|
|
14604
15061
|
if (!this.targetResource) {
|
|
14605
15062
|
throw new Error(`S3QueuePlugin: resource '${this.config.resource}' not found`);
|
|
@@ -15160,8 +15617,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
15160
15617
|
if (parts.length !== 5) return false;
|
|
15161
15618
|
return true;
|
|
15162
15619
|
}
|
|
15163
|
-
async
|
|
15164
|
-
this.database = database;
|
|
15620
|
+
async onInstall() {
|
|
15165
15621
|
await this._createLockResource();
|
|
15166
15622
|
if (this.config.persistJobs) {
|
|
15167
15623
|
await this._createJobHistoryResource();
|
|
@@ -15719,8 +16175,7 @@ class StateMachinePlugin extends Plugin {
|
|
|
15719
16175
|
}
|
|
15720
16176
|
}
|
|
15721
16177
|
}
|
|
15722
|
-
async
|
|
15723
|
-
this.database = database;
|
|
16178
|
+
async onInstall() {
|
|
15724
16179
|
if (this.config.persistTransitions) {
|
|
15725
16180
|
await this._createStateResources();
|
|
15726
16181
|
}
|