s3db.js 7.3.3 → 7.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +57 -146
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +44 -133
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +57 -147
- package/dist/s3db.iife.min.js +1 -1
- package/package.json +24 -24
- package/src/database.class.js +10 -8
- package/src/plugins/cache/filesystem-cache.class.js +1 -8
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +1 -8
- package/src/plugins/metrics.plugin.js +18 -8
- package/src/plugins/replicator.plugin.js +0 -26
- package/src/plugins/replicators/s3db-replicator.class.js +3 -3
package/dist/s3db.es.js
CHANGED
|
@@ -2,8 +2,9 @@ import { customAlphabet, urlAlphabet } from 'nanoid';
|
|
|
2
2
|
import zlib from 'node:zlib';
|
|
3
3
|
import { PromisePool } from '@supercharge/promise-pool';
|
|
4
4
|
import { ReadableStream } from 'node:stream/web';
|
|
5
|
-
import {
|
|
5
|
+
import { mkdir, writeFile, readFile, stat, unlink, readdir, rm } from 'fs/promises';
|
|
6
6
|
import { createHash } from 'crypto';
|
|
7
|
+
import { chunk, merge, isString as isString$1, isEmpty, invert, uniq, cloneDeep, get, set, isObject as isObject$1, isFunction as isFunction$1 } from 'lodash-es';
|
|
7
8
|
import jsonStableStringify from 'json-stable-stringify';
|
|
8
9
|
import { S3Client, PutObjectCommand, GetObjectCommand, HeadObjectCommand, CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
|
|
9
10
|
import { flatten, unflatten } from 'flat';
|
|
@@ -2077,16 +2078,6 @@ if (typeof Object.create === 'function'){
|
|
|
2077
2078
|
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
2078
2079
|
// USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
2079
2080
|
|
|
2080
|
-
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
2081
|
-
function getOwnPropertyDescriptors(obj) {
|
|
2082
|
-
var keys = Object.keys(obj);
|
|
2083
|
-
var descriptors = {};
|
|
2084
|
-
for (var i = 0; i < keys.length; i++) {
|
|
2085
|
-
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
2086
|
-
}
|
|
2087
|
-
return descriptors;
|
|
2088
|
-
};
|
|
2089
|
-
|
|
2090
2081
|
var formatRegExp = /%[sdj%]/g;
|
|
2091
2082
|
function format(f) {
|
|
2092
2083
|
if (!isString(f)) {
|
|
@@ -2572,64 +2563,6 @@ function hasOwnProperty(obj, prop) {
|
|
|
2572
2563
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
2573
2564
|
}
|
|
2574
2565
|
|
|
2575
|
-
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
2576
|
-
|
|
2577
|
-
function promisify(original) {
|
|
2578
|
-
if (typeof original !== 'function')
|
|
2579
|
-
throw new TypeError('The "original" argument must be of type Function');
|
|
2580
|
-
|
|
2581
|
-
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
2582
|
-
var fn = original[kCustomPromisifiedSymbol];
|
|
2583
|
-
if (typeof fn !== 'function') {
|
|
2584
|
-
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
2585
|
-
}
|
|
2586
|
-
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2587
|
-
value: fn, enumerable: false, writable: false, configurable: true
|
|
2588
|
-
});
|
|
2589
|
-
return fn;
|
|
2590
|
-
}
|
|
2591
|
-
|
|
2592
|
-
function fn() {
|
|
2593
|
-
var promiseResolve, promiseReject;
|
|
2594
|
-
var promise = new Promise(function (resolve, reject) {
|
|
2595
|
-
promiseResolve = resolve;
|
|
2596
|
-
promiseReject = reject;
|
|
2597
|
-
});
|
|
2598
|
-
|
|
2599
|
-
var args = [];
|
|
2600
|
-
for (var i = 0; i < arguments.length; i++) {
|
|
2601
|
-
args.push(arguments[i]);
|
|
2602
|
-
}
|
|
2603
|
-
args.push(function (err, value) {
|
|
2604
|
-
if (err) {
|
|
2605
|
-
promiseReject(err);
|
|
2606
|
-
} else {
|
|
2607
|
-
promiseResolve(value);
|
|
2608
|
-
}
|
|
2609
|
-
});
|
|
2610
|
-
|
|
2611
|
-
try {
|
|
2612
|
-
original.apply(this, args);
|
|
2613
|
-
} catch (err) {
|
|
2614
|
-
promiseReject(err);
|
|
2615
|
-
}
|
|
2616
|
-
|
|
2617
|
-
return promise;
|
|
2618
|
-
}
|
|
2619
|
-
|
|
2620
|
-
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
2621
|
-
|
|
2622
|
-
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2623
|
-
value: fn, enumerable: false, writable: false, configurable: true
|
|
2624
|
-
});
|
|
2625
|
-
return Object.defineProperties(
|
|
2626
|
-
fn,
|
|
2627
|
-
getOwnPropertyDescriptors(original)
|
|
2628
|
-
);
|
|
2629
|
-
}
|
|
2630
|
-
|
|
2631
|
-
promisify.custom = kCustomPromisifiedSymbol;
|
|
2632
|
-
|
|
2633
2566
|
var lookup = [];
|
|
2634
2567
|
var revLookup = [];
|
|
2635
2568
|
var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array;
|
|
@@ -6882,12 +6815,6 @@ var memory_cache_class_default = MemoryCache;
|
|
|
6882
6815
|
|
|
6883
6816
|
var fs = {};
|
|
6884
6817
|
|
|
6885
|
-
const readFile$1 = promisify(fs.readFile);
|
|
6886
|
-
const writeFile$1 = promisify(fs.writeFile);
|
|
6887
|
-
const unlink = promisify(fs.unlink);
|
|
6888
|
-
const readdir$1 = promisify(fs.readdir);
|
|
6889
|
-
const stat$1 = promisify(fs.stat);
|
|
6890
|
-
const mkdir = promisify(fs.mkdir);
|
|
6891
6818
|
class FilesystemCache extends Cache {
|
|
6892
6819
|
constructor({
|
|
6893
6820
|
directory,
|
|
@@ -7001,7 +6928,7 @@ class FilesystemCache extends Cache {
|
|
|
7001
6928
|
await this._acquireLock(filePath);
|
|
7002
6929
|
}
|
|
7003
6930
|
try {
|
|
7004
|
-
await writeFile
|
|
6931
|
+
await writeFile(filePath, finalData, {
|
|
7005
6932
|
encoding: compressed ? "utf8" : this.encoding,
|
|
7006
6933
|
mode: this.fileMode
|
|
7007
6934
|
});
|
|
@@ -7015,7 +6942,7 @@ class FilesystemCache extends Cache {
|
|
|
7015
6942
|
compressedSize: compressed ? Buffer.byteLength(finalData, "utf8") : originalSize,
|
|
7016
6943
|
compressionRatio: compressed ? (Buffer.byteLength(finalData, "utf8") / originalSize).toFixed(2) : 1
|
|
7017
6944
|
};
|
|
7018
|
-
await writeFile
|
|
6945
|
+
await writeFile(this._getMetadataPath(filePath), JSON.stringify(metadata), {
|
|
7019
6946
|
encoding: this.encoding,
|
|
7020
6947
|
mode: this.fileMode
|
|
7021
6948
|
});
|
|
@@ -7053,7 +6980,7 @@ class FilesystemCache extends Cache {
|
|
|
7053
6980
|
const metadataPath = this._getMetadataPath(filePath);
|
|
7054
6981
|
if (await this._fileExists(metadataPath)) {
|
|
7055
6982
|
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7056
|
-
const metaContent = await readFile
|
|
6983
|
+
const metaContent = await readFile(metadataPath, this.encoding);
|
|
7057
6984
|
return JSON.parse(metaContent);
|
|
7058
6985
|
});
|
|
7059
6986
|
if (ok && metadata.ttl > 0) {
|
|
@@ -7062,7 +6989,7 @@ class FilesystemCache extends Cache {
|
|
|
7062
6989
|
}
|
|
7063
6990
|
}
|
|
7064
6991
|
} else if (this.ttl > 0) {
|
|
7065
|
-
const stats = await stat
|
|
6992
|
+
const stats = await stat(filePath);
|
|
7066
6993
|
const age = Date.now() - stats.mtime.getTime();
|
|
7067
6994
|
isExpired = age > this.ttl;
|
|
7068
6995
|
}
|
|
@@ -7077,13 +7004,13 @@ class FilesystemCache extends Cache {
|
|
|
7077
7004
|
await this._acquireLock(filePath);
|
|
7078
7005
|
}
|
|
7079
7006
|
try {
|
|
7080
|
-
const content = await readFile
|
|
7007
|
+
const content = await readFile(filePath, this.encoding);
|
|
7081
7008
|
let isCompressed = false;
|
|
7082
7009
|
if (this.enableMetadata) {
|
|
7083
7010
|
const metadataPath = this._getMetadataPath(filePath);
|
|
7084
7011
|
if (await this._fileExists(metadataPath)) {
|
|
7085
7012
|
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7086
|
-
const metaContent = await readFile
|
|
7013
|
+
const metaContent = await readFile(metadataPath, this.encoding);
|
|
7087
7014
|
return JSON.parse(metaContent);
|
|
7088
7015
|
});
|
|
7089
7016
|
if (ok) {
|
|
@@ -7152,7 +7079,7 @@ class FilesystemCache extends Cache {
|
|
|
7152
7079
|
}
|
|
7153
7080
|
async _clear(prefix) {
|
|
7154
7081
|
try {
|
|
7155
|
-
const files = await readdir
|
|
7082
|
+
const files = await readdir(this.directory);
|
|
7156
7083
|
const cacheFiles = files.filter((file) => {
|
|
7157
7084
|
if (!file.startsWith(this.prefix)) return false;
|
|
7158
7085
|
if (!file.endsWith(this.fileExtension)) return false;
|
|
@@ -7200,7 +7127,7 @@ class FilesystemCache extends Cache {
|
|
|
7200
7127
|
}
|
|
7201
7128
|
async keys() {
|
|
7202
7129
|
try {
|
|
7203
|
-
const files = await readdir
|
|
7130
|
+
const files = await readdir(this.directory);
|
|
7204
7131
|
const cacheFiles = files.filter(
|
|
7205
7132
|
(file) => file.startsWith(this.prefix) && file.endsWith(this.fileExtension)
|
|
7206
7133
|
);
|
|
@@ -7217,14 +7144,14 @@ class FilesystemCache extends Cache {
|
|
|
7217
7144
|
// Helper methods
|
|
7218
7145
|
async _fileExists(filePath) {
|
|
7219
7146
|
const [ok] = await try_fn_default(async () => {
|
|
7220
|
-
await stat
|
|
7147
|
+
await stat(filePath);
|
|
7221
7148
|
});
|
|
7222
7149
|
return ok;
|
|
7223
7150
|
}
|
|
7224
7151
|
async _copyFile(src, dest) {
|
|
7225
7152
|
const [ok, err] = await try_fn_default(async () => {
|
|
7226
|
-
const content = await readFile
|
|
7227
|
-
await writeFile
|
|
7153
|
+
const content = await readFile(src);
|
|
7154
|
+
await writeFile(dest, content);
|
|
7228
7155
|
});
|
|
7229
7156
|
if (!ok) {
|
|
7230
7157
|
console.warn("FilesystemCache: Failed to create backup:", err.message);
|
|
@@ -7233,7 +7160,7 @@ class FilesystemCache extends Cache {
|
|
|
7233
7160
|
async _cleanup() {
|
|
7234
7161
|
if (!this.ttl || this.ttl <= 0) return;
|
|
7235
7162
|
try {
|
|
7236
|
-
const files = await readdir
|
|
7163
|
+
const files = await readdir(this.directory);
|
|
7237
7164
|
const now = Date.now();
|
|
7238
7165
|
for (const file of files) {
|
|
7239
7166
|
if (!file.startsWith(this.prefix) || !file.endsWith(this.fileExtension)) {
|
|
@@ -7245,7 +7172,7 @@ class FilesystemCache extends Cache {
|
|
|
7245
7172
|
const metadataPath = this._getMetadataPath(filePath);
|
|
7246
7173
|
if (await this._fileExists(metadataPath)) {
|
|
7247
7174
|
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7248
|
-
const metaContent = await readFile
|
|
7175
|
+
const metaContent = await readFile(metadataPath, this.encoding);
|
|
7249
7176
|
return JSON.parse(metaContent);
|
|
7250
7177
|
});
|
|
7251
7178
|
if (ok && metadata.ttl > 0) {
|
|
@@ -7255,7 +7182,7 @@ class FilesystemCache extends Cache {
|
|
|
7255
7182
|
}
|
|
7256
7183
|
} else {
|
|
7257
7184
|
const [ok, err, stats] = await try_fn_default(async () => {
|
|
7258
|
-
return await stat
|
|
7185
|
+
return await stat(filePath);
|
|
7259
7186
|
});
|
|
7260
7187
|
if (ok) {
|
|
7261
7188
|
const age = now - stats.mtime.getTime();
|
|
@@ -7325,12 +7252,6 @@ class FilesystemCache extends Cache {
|
|
|
7325
7252
|
}
|
|
7326
7253
|
}
|
|
7327
7254
|
|
|
7328
|
-
promisify(fs.mkdir);
|
|
7329
|
-
const rmdir = promisify(fs.rm);
|
|
7330
|
-
const readdir = promisify(fs.readdir);
|
|
7331
|
-
const stat = promisify(fs.stat);
|
|
7332
|
-
const writeFile = promisify(fs.writeFile);
|
|
7333
|
-
const readFile = promisify(fs.readFile);
|
|
7334
7255
|
class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
7335
7256
|
constructor({
|
|
7336
7257
|
partitionStrategy = "hierarchical",
|
|
@@ -7452,7 +7373,7 @@ class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
|
7452
7373
|
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7453
7374
|
const [ok, err] = await try_fn_default(async () => {
|
|
7454
7375
|
if (await this._fileExists(partitionDir)) {
|
|
7455
|
-
await
|
|
7376
|
+
await rm(partitionDir, { recursive: true });
|
|
7456
7377
|
}
|
|
7457
7378
|
});
|
|
7458
7379
|
if (!ok) {
|
|
@@ -7470,7 +7391,7 @@ class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
|
7470
7391
|
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7471
7392
|
const [ok, err] = await try_fn_default(async () => {
|
|
7472
7393
|
if (await this._fileExists(resourceDir)) {
|
|
7473
|
-
await
|
|
7394
|
+
await rm(resourceDir, { recursive: true });
|
|
7474
7395
|
}
|
|
7475
7396
|
});
|
|
7476
7397
|
for (const [key] of this.partitionUsage.entries()) {
|
|
@@ -8428,7 +8349,7 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
8428
8349
|
}
|
|
8429
8350
|
async setup(database) {
|
|
8430
8351
|
this.database = database;
|
|
8431
|
-
if (process.env.NODE_ENV === "test") return;
|
|
8352
|
+
if (typeof process !== "undefined" && process.env.NODE_ENV === "test") return;
|
|
8432
8353
|
const [ok, err] = await try_fn_default(async () => {
|
|
8433
8354
|
const [ok1, err1, metricsResource] = await try_fn_default(() => database.createResource({
|
|
8434
8355
|
name: "metrics",
|
|
@@ -8478,7 +8399,7 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
8478
8399
|
this.performanceResource = database.resources.performance_logs;
|
|
8479
8400
|
}
|
|
8480
8401
|
this.installMetricsHooks();
|
|
8481
|
-
if (process.env.NODE_ENV !== "test") {
|
|
8402
|
+
if (typeof process !== "undefined" && process.env.NODE_ENV !== "test") {
|
|
8482
8403
|
this.startFlushTimer();
|
|
8483
8404
|
}
|
|
8484
8405
|
}
|
|
@@ -8489,7 +8410,7 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
8489
8410
|
clearInterval(this.flushTimer);
|
|
8490
8411
|
this.flushTimer = null;
|
|
8491
8412
|
}
|
|
8492
|
-
if (process.env.NODE_ENV !== "test") {
|
|
8413
|
+
if (typeof process !== "undefined" && process.env.NODE_ENV !== "test") {
|
|
8493
8414
|
await this.flushMetrics();
|
|
8494
8415
|
}
|
|
8495
8416
|
}
|
|
@@ -8668,10 +8589,18 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
8668
8589
|
async flushMetrics() {
|
|
8669
8590
|
if (!this.metricsResource) return;
|
|
8670
8591
|
const [ok, err] = await try_fn_default(async () => {
|
|
8671
|
-
|
|
8672
|
-
|
|
8673
|
-
|
|
8674
|
-
|
|
8592
|
+
let metadata, perfMetadata, errorMetadata, resourceMetadata;
|
|
8593
|
+
if (typeof process !== "undefined" && process.env.NODE_ENV === "test") {
|
|
8594
|
+
metadata = {};
|
|
8595
|
+
perfMetadata = {};
|
|
8596
|
+
errorMetadata = {};
|
|
8597
|
+
resourceMetadata = {};
|
|
8598
|
+
} else {
|
|
8599
|
+
metadata = { global: "true" };
|
|
8600
|
+
perfMetadata = { perf: "true" };
|
|
8601
|
+
errorMetadata = { error: "true" };
|
|
8602
|
+
resourceMetadata = { resource: "true" };
|
|
8603
|
+
}
|
|
8675
8604
|
for (const [operation, data] of Object.entries(this.metrics.operations)) {
|
|
8676
8605
|
if (data.count > 0) {
|
|
8677
8606
|
await this.metricsResource.insert({
|
|
@@ -13204,7 +13133,7 @@ class Database extends EventEmitter {
|
|
|
13204
13133
|
super();
|
|
13205
13134
|
this.version = "1";
|
|
13206
13135
|
this.s3dbVersion = (() => {
|
|
13207
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.3.
|
|
13136
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.4" : "latest");
|
|
13208
13137
|
return ok ? version : "latest";
|
|
13209
13138
|
})();
|
|
13210
13139
|
this.resources = {};
|
|
@@ -13247,14 +13176,16 @@ class Database extends EventEmitter {
|
|
|
13247
13176
|
this.keyPrefix = this.client.keyPrefix;
|
|
13248
13177
|
if (!this._exitListenerRegistered) {
|
|
13249
13178
|
this._exitListenerRegistered = true;
|
|
13250
|
-
process
|
|
13251
|
-
|
|
13252
|
-
|
|
13253
|
-
|
|
13254
|
-
|
|
13179
|
+
if (typeof process !== "undefined") {
|
|
13180
|
+
process.on("exit", async () => {
|
|
13181
|
+
if (this.isConnected()) {
|
|
13182
|
+
try {
|
|
13183
|
+
await this.disconnect();
|
|
13184
|
+
} catch (err) {
|
|
13185
|
+
}
|
|
13255
13186
|
}
|
|
13256
|
-
}
|
|
13257
|
-
}
|
|
13187
|
+
});
|
|
13188
|
+
}
|
|
13258
13189
|
}
|
|
13259
13190
|
}
|
|
13260
13191
|
async connect() {
|
|
@@ -13837,7 +13768,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13837
13768
|
if (typeof entry[0] === "function") return resource;
|
|
13838
13769
|
}
|
|
13839
13770
|
if (typeof entry === "string") return entry;
|
|
13840
|
-
if (
|
|
13771
|
+
if (typeof entry === "function") return resource;
|
|
13841
13772
|
if (typeof entry === "object" && entry.resource) return entry.resource;
|
|
13842
13773
|
return resource;
|
|
13843
13774
|
}
|
|
@@ -14386,31 +14317,11 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14386
14317
|
}
|
|
14387
14318
|
async stop() {
|
|
14388
14319
|
}
|
|
14389
|
-
filterInternalFields(data) {
|
|
14390
|
-
if (!data || typeof data !== "object") return data;
|
|
14391
|
-
const filtered = {};
|
|
14392
|
-
for (const [key, value] of Object.entries(data)) {
|
|
14393
|
-
if (!key.startsWith("_") && !key.startsWith("$")) {
|
|
14394
|
-
filtered[key] = value;
|
|
14395
|
-
}
|
|
14396
|
-
}
|
|
14397
|
-
return filtered;
|
|
14398
|
-
}
|
|
14399
14320
|
async uploadMetadataFile(database) {
|
|
14400
14321
|
if (typeof database.uploadMetadataFile === "function") {
|
|
14401
14322
|
await database.uploadMetadataFile();
|
|
14402
14323
|
}
|
|
14403
14324
|
}
|
|
14404
|
-
async getCompleteData(resource, data) {
|
|
14405
|
-
try {
|
|
14406
|
-
const [ok, err, record] = await try_fn_default(() => resource.get(data.id));
|
|
14407
|
-
if (ok && record) {
|
|
14408
|
-
return record;
|
|
14409
|
-
}
|
|
14410
|
-
} catch (error) {
|
|
14411
|
-
}
|
|
14412
|
-
return data;
|
|
14413
|
-
}
|
|
14414
14325
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14415
14326
|
let lastError;
|
|
14416
14327
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|