s3db.js 11.0.4 → 11.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -2
- package/dist/s3db.cjs.js +502 -84
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +502 -84
- package/dist/s3db.es.js.map +1 -1
- package/package.json +2 -1
- package/src/concerns/crypto.js +7 -14
- package/src/plugins/eventual-consistency/analytics.js +44 -22
- package/src/plugins/eventual-consistency/consolidation.js +228 -80
- package/src/plugins/eventual-consistency/helpers.js +24 -8
- package/src/plugins/eventual-consistency/index.js +73 -1
- package/src/plugins/eventual-consistency/install.js +1 -0
- package/src/plugins/eventual-consistency/utils.js +154 -4
- package/src/concerns/advanced-metadata-encoding.js +0 -440
package/dist/s3db.cjs.js
CHANGED
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
4
|
|
|
5
|
+
var crypto = require('crypto');
|
|
5
6
|
var nanoid = require('nanoid');
|
|
6
7
|
var EventEmitter = require('events');
|
|
7
8
|
var promises = require('fs/promises');
|
|
8
9
|
var fs = require('fs');
|
|
9
10
|
var promises$1 = require('stream/promises');
|
|
10
11
|
var path = require('path');
|
|
11
|
-
var crypto = require('crypto');
|
|
12
12
|
var zlib = require('node:zlib');
|
|
13
13
|
var os = require('os');
|
|
14
14
|
var jsonStableStringify = require('json-stable-stringify');
|
|
@@ -528,15 +528,7 @@ function tryFnSync(fn) {
|
|
|
528
528
|
async function dynamicCrypto() {
|
|
529
529
|
let lib;
|
|
530
530
|
if (typeof process !== "undefined") {
|
|
531
|
-
|
|
532
|
-
const { webcrypto } = await import('crypto');
|
|
533
|
-
return webcrypto;
|
|
534
|
-
});
|
|
535
|
-
if (ok) {
|
|
536
|
-
lib = result;
|
|
537
|
-
} else {
|
|
538
|
-
throw new CryptoError("Crypto API not available", { original: err, context: "dynamicCrypto" });
|
|
539
|
-
}
|
|
531
|
+
lib = crypto.webcrypto;
|
|
540
532
|
} else if (typeof window !== "undefined") {
|
|
541
533
|
lib = window.crypto;
|
|
542
534
|
}
|
|
@@ -590,8 +582,7 @@ async function md5(data) {
|
|
|
590
582
|
throw new CryptoError("MD5 hashing is only available in Node.js environment", { context: "md5" });
|
|
591
583
|
}
|
|
592
584
|
const [ok, err, result] = await tryFn(async () => {
|
|
593
|
-
|
|
594
|
-
return createHash("md5").update(data).digest("base64");
|
|
585
|
+
return crypto.createHash("md5").update(data).digest("base64");
|
|
595
586
|
});
|
|
596
587
|
if (!ok) {
|
|
597
588
|
throw new CryptoError("MD5 hashing failed", { original: err, data });
|
|
@@ -5276,10 +5267,121 @@ function createFieldHandler(resourceName, fieldName) {
|
|
|
5276
5267
|
deferredSetup: false
|
|
5277
5268
|
};
|
|
5278
5269
|
}
|
|
5270
|
+
function validateNestedPath(resource, fieldPath) {
|
|
5271
|
+
const parts = fieldPath.split(".");
|
|
5272
|
+
const rootField = parts[0];
|
|
5273
|
+
if (!resource.attributes || !resource.attributes[rootField]) {
|
|
5274
|
+
return {
|
|
5275
|
+
valid: false,
|
|
5276
|
+
rootField,
|
|
5277
|
+
fullPath: fieldPath,
|
|
5278
|
+
error: `Root field "${rootField}" not found in resource attributes`
|
|
5279
|
+
};
|
|
5280
|
+
}
|
|
5281
|
+
if (parts.length === 1) {
|
|
5282
|
+
return { valid: true, rootField, fullPath: fieldPath };
|
|
5283
|
+
}
|
|
5284
|
+
let current = resource.attributes[rootField];
|
|
5285
|
+
let foundJson = false;
|
|
5286
|
+
let levelsAfterJson = 0;
|
|
5287
|
+
for (let i = 1; i < parts.length; i++) {
|
|
5288
|
+
const part = parts[i];
|
|
5289
|
+
if (foundJson) {
|
|
5290
|
+
levelsAfterJson++;
|
|
5291
|
+
if (levelsAfterJson > 1) {
|
|
5292
|
+
return {
|
|
5293
|
+
valid: false,
|
|
5294
|
+
rootField,
|
|
5295
|
+
fullPath: fieldPath,
|
|
5296
|
+
error: `Path "${fieldPath}" exceeds 1 level after 'json' field. Maximum nesting after 'json' is 1 level.`
|
|
5297
|
+
};
|
|
5298
|
+
}
|
|
5299
|
+
continue;
|
|
5300
|
+
}
|
|
5301
|
+
if (typeof current === "string") {
|
|
5302
|
+
if (current === "json" || current.startsWith("json|")) {
|
|
5303
|
+
foundJson = true;
|
|
5304
|
+
levelsAfterJson++;
|
|
5305
|
+
if (levelsAfterJson > 1) {
|
|
5306
|
+
return {
|
|
5307
|
+
valid: false,
|
|
5308
|
+
rootField,
|
|
5309
|
+
fullPath: fieldPath,
|
|
5310
|
+
error: `Path "${fieldPath}" exceeds 1 level after 'json' field`
|
|
5311
|
+
};
|
|
5312
|
+
}
|
|
5313
|
+
continue;
|
|
5314
|
+
}
|
|
5315
|
+
return {
|
|
5316
|
+
valid: false,
|
|
5317
|
+
rootField,
|
|
5318
|
+
fullPath: fieldPath,
|
|
5319
|
+
error: `Field "${parts.slice(0, i).join(".")}" is type "${current}" and cannot be nested`
|
|
5320
|
+
};
|
|
5321
|
+
}
|
|
5322
|
+
if (typeof current === "object") {
|
|
5323
|
+
if (current.$$type) {
|
|
5324
|
+
const type = current.$$type;
|
|
5325
|
+
if (type === "json" || type.includes("json")) {
|
|
5326
|
+
foundJson = true;
|
|
5327
|
+
levelsAfterJson++;
|
|
5328
|
+
continue;
|
|
5329
|
+
}
|
|
5330
|
+
if (type !== "object" && !type.includes("object")) {
|
|
5331
|
+
return {
|
|
5332
|
+
valid: false,
|
|
5333
|
+
rootField,
|
|
5334
|
+
fullPath: fieldPath,
|
|
5335
|
+
error: `Field "${parts.slice(0, i).join(".")}" is type "${type}" and cannot be nested`
|
|
5336
|
+
};
|
|
5337
|
+
}
|
|
5338
|
+
}
|
|
5339
|
+
if (!current[part]) {
|
|
5340
|
+
return {
|
|
5341
|
+
valid: false,
|
|
5342
|
+
rootField,
|
|
5343
|
+
fullPath: fieldPath,
|
|
5344
|
+
error: `Field "${part}" not found in "${parts.slice(0, i).join(".")}"`
|
|
5345
|
+
};
|
|
5346
|
+
}
|
|
5347
|
+
current = current[part];
|
|
5348
|
+
} else {
|
|
5349
|
+
return {
|
|
5350
|
+
valid: false,
|
|
5351
|
+
rootField,
|
|
5352
|
+
fullPath: fieldPath,
|
|
5353
|
+
error: `Invalid structure at "${parts.slice(0, i).join(".")}"`
|
|
5354
|
+
};
|
|
5355
|
+
}
|
|
5356
|
+
}
|
|
5357
|
+
return { valid: true, rootField, fullPath: fieldPath };
|
|
5358
|
+
}
|
|
5279
5359
|
function resolveFieldAndPlugin(resource, field, value) {
|
|
5280
5360
|
if (!resource._eventualConsistencyPlugins) {
|
|
5281
5361
|
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
5282
5362
|
}
|
|
5363
|
+
if (field.includes(".")) {
|
|
5364
|
+
const validation = validateNestedPath(resource, field);
|
|
5365
|
+
if (!validation.valid) {
|
|
5366
|
+
throw new Error(validation.error);
|
|
5367
|
+
}
|
|
5368
|
+
const rootField = validation.rootField;
|
|
5369
|
+
const fieldPlugin2 = resource._eventualConsistencyPlugins[rootField];
|
|
5370
|
+
if (!fieldPlugin2) {
|
|
5371
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5372
|
+
throw new Error(
|
|
5373
|
+
`No eventual consistency plugin found for root field "${rootField}". Available fields: ${availableFields}`
|
|
5374
|
+
);
|
|
5375
|
+
}
|
|
5376
|
+
return {
|
|
5377
|
+
field: rootField,
|
|
5378
|
+
// Root field for plugin lookup
|
|
5379
|
+
fieldPath: field,
|
|
5380
|
+
// Full path for nested access
|
|
5381
|
+
value,
|
|
5382
|
+
plugin: fieldPlugin2
|
|
5383
|
+
};
|
|
5384
|
+
}
|
|
5283
5385
|
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
5284
5386
|
if (!fieldPlugin) {
|
|
5285
5387
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
@@ -5287,7 +5389,7 @@ function resolveFieldAndPlugin(resource, field, value) {
|
|
|
5287
5389
|
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5288
5390
|
);
|
|
5289
5391
|
}
|
|
5290
|
-
return { field, value, plugin: fieldPlugin };
|
|
5392
|
+
return { field, fieldPath: field, value, plugin: fieldPlugin };
|
|
5291
5393
|
}
|
|
5292
5394
|
function groupByCohort(transactions, cohortField) {
|
|
5293
5395
|
const groups = {};
|
|
@@ -5532,10 +5634,10 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5532
5634
|
})
|
|
5533
5635
|
);
|
|
5534
5636
|
if (!ok || !transactions || transactions.length === 0) {
|
|
5535
|
-
const [
|
|
5637
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
5536
5638
|
() => targetResource.get(originalId)
|
|
5537
5639
|
);
|
|
5538
|
-
const currentValue2 =
|
|
5640
|
+
const currentValue2 = recordOk2 && record2 ? record2[config.field] || 0 : 0;
|
|
5539
5641
|
if (config.verbose) {
|
|
5540
5642
|
console.log(
|
|
5541
5643
|
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions for ${originalId}, skipping`
|
|
@@ -5570,6 +5672,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5570
5672
|
);
|
|
5571
5673
|
}
|
|
5572
5674
|
currentValue = 0;
|
|
5675
|
+
appliedTransactions.length = 0;
|
|
5573
5676
|
} else {
|
|
5574
5677
|
appliedTransactions.sort(
|
|
5575
5678
|
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
@@ -5577,42 +5680,46 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5577
5680
|
const hasSetInApplied = appliedTransactions.some((t) => t.operation === "set");
|
|
5578
5681
|
if (!hasSetInApplied) {
|
|
5579
5682
|
const recordValue = recordExists[config.field] || 0;
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
|
|
5593
|
-
|
|
5594
|
-
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5683
|
+
if (typeof recordValue === "number") {
|
|
5684
|
+
let appliedDelta = 0;
|
|
5685
|
+
for (const t of appliedTransactions) {
|
|
5686
|
+
if (t.operation === "add") appliedDelta += t.value;
|
|
5687
|
+
else if (t.operation === "sub") appliedDelta -= t.value;
|
|
5688
|
+
}
|
|
5689
|
+
const baseValue = recordValue - appliedDelta;
|
|
5690
|
+
const hasExistingAnchor = appliedTransactions.some((t) => t.source === "anchor");
|
|
5691
|
+
if (baseValue !== 0 && typeof baseValue === "number" && !hasExistingAnchor) {
|
|
5692
|
+
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
5693
|
+
const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
|
|
5694
|
+
const anchorTransaction = {
|
|
5695
|
+
id: idGenerator(),
|
|
5696
|
+
originalId,
|
|
5697
|
+
field: config.field,
|
|
5698
|
+
fieldPath: config.field,
|
|
5699
|
+
// Add fieldPath for consistency
|
|
5700
|
+
value: baseValue,
|
|
5701
|
+
operation: "set",
|
|
5702
|
+
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(),
|
|
5703
|
+
// 1ms before first txn to ensure it's first
|
|
5704
|
+
cohortDate: cohortInfo.date,
|
|
5705
|
+
cohortHour: cohortInfo.hour,
|
|
5706
|
+
cohortMonth: cohortInfo.month,
|
|
5707
|
+
source: "anchor",
|
|
5708
|
+
applied: true
|
|
5709
|
+
};
|
|
5710
|
+
await transactionResource.insert(anchorTransaction);
|
|
5711
|
+
appliedTransactions.unshift(anchorTransaction);
|
|
5712
|
+
}
|
|
5606
5713
|
}
|
|
5607
5714
|
}
|
|
5608
5715
|
currentValue = config.reducer(appliedTransactions);
|
|
5609
5716
|
}
|
|
5610
5717
|
} else {
|
|
5611
|
-
const [
|
|
5718
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
5612
5719
|
() => targetResource.get(originalId)
|
|
5613
5720
|
);
|
|
5614
|
-
currentValue =
|
|
5615
|
-
if (currentValue !== 0) {
|
|
5721
|
+
currentValue = recordOk2 && record2 ? record2[config.field] || 0 : 0;
|
|
5722
|
+
if (currentValue !== 0 && typeof currentValue === "number") {
|
|
5616
5723
|
let anchorTimestamp;
|
|
5617
5724
|
if (transactions && transactions.length > 0) {
|
|
5618
5725
|
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
@@ -5625,6 +5732,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5625
5732
|
id: idGenerator(),
|
|
5626
5733
|
originalId,
|
|
5627
5734
|
field: config.field,
|
|
5735
|
+
fieldPath: config.field,
|
|
5736
|
+
// Add fieldPath for consistency
|
|
5628
5737
|
value: currentValue,
|
|
5629
5738
|
operation: "set",
|
|
5630
5739
|
timestamp: anchorTimestamp,
|
|
@@ -5650,38 +5759,101 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5650
5759
|
transactions.sort(
|
|
5651
5760
|
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5652
5761
|
);
|
|
5653
|
-
const
|
|
5654
|
-
|
|
5655
|
-
|
|
5762
|
+
const transactionsByPath = {};
|
|
5763
|
+
for (const txn of transactions) {
|
|
5764
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
5765
|
+
if (!transactionsByPath[path]) {
|
|
5766
|
+
transactionsByPath[path] = [];
|
|
5767
|
+
}
|
|
5768
|
+
transactionsByPath[path].push(txn);
|
|
5656
5769
|
}
|
|
5657
|
-
const
|
|
5658
|
-
if (
|
|
5659
|
-
|
|
5660
|
-
|
|
5661
|
-
|
|
5770
|
+
const appliedByPath = {};
|
|
5771
|
+
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
5772
|
+
for (const txn of appliedTransactions) {
|
|
5773
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
5774
|
+
if (!appliedByPath[path]) {
|
|
5775
|
+
appliedByPath[path] = [];
|
|
5776
|
+
}
|
|
5777
|
+
appliedByPath[path].push(txn);
|
|
5778
|
+
}
|
|
5779
|
+
}
|
|
5780
|
+
const consolidatedValues = {};
|
|
5781
|
+
const lodash = await import('lodash-es');
|
|
5782
|
+
const [currentRecordOk, currentRecordErr, currentRecord] = await tryFn(
|
|
5783
|
+
() => targetResource.get(originalId)
|
|
5784
|
+
);
|
|
5785
|
+
for (const [fieldPath, pathTransactions] of Object.entries(transactionsByPath)) {
|
|
5786
|
+
let pathCurrentValue = 0;
|
|
5787
|
+
if (appliedByPath[fieldPath] && appliedByPath[fieldPath].length > 0) {
|
|
5788
|
+
appliedByPath[fieldPath].sort(
|
|
5789
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5790
|
+
);
|
|
5791
|
+
pathCurrentValue = config.reducer(appliedByPath[fieldPath]);
|
|
5792
|
+
} else {
|
|
5793
|
+
if (currentRecordOk && currentRecord) {
|
|
5794
|
+
const recordValue = lodash.get(currentRecord, fieldPath, 0);
|
|
5795
|
+
if (typeof recordValue === "number") {
|
|
5796
|
+
pathCurrentValue = recordValue;
|
|
5797
|
+
}
|
|
5798
|
+
}
|
|
5799
|
+
}
|
|
5800
|
+
if (pathCurrentValue !== 0) {
|
|
5801
|
+
pathTransactions.unshift(createSyntheticSetTransaction(pathCurrentValue));
|
|
5802
|
+
}
|
|
5803
|
+
const pathConsolidatedValue = config.reducer(pathTransactions);
|
|
5804
|
+
consolidatedValues[fieldPath] = pathConsolidatedValue;
|
|
5805
|
+
if (config.verbose) {
|
|
5806
|
+
console.log(
|
|
5807
|
+
`[EventualConsistency] ${config.resource}.${fieldPath} - ${originalId}: ${pathCurrentValue} \u2192 ${pathConsolidatedValue} (${pathTransactions.length - (pathCurrentValue !== 0 ? 1 : 0)} pending txns)`
|
|
5808
|
+
);
|
|
5809
|
+
}
|
|
5662
5810
|
}
|
|
5663
5811
|
if (config.verbose) {
|
|
5664
5812
|
console.log(
|
|
5665
5813
|
`\u{1F525} [DEBUG] BEFORE targetResource.update() {
|
|
5666
5814
|
originalId: '${originalId}',
|
|
5667
|
-
|
|
5668
|
-
consolidatedValue: ${consolidatedValue},
|
|
5669
|
-
currentValue: ${currentValue}
|
|
5815
|
+
consolidatedValues: ${JSON.stringify(consolidatedValues, null, 2)}
|
|
5670
5816
|
}`
|
|
5671
5817
|
);
|
|
5672
5818
|
}
|
|
5673
|
-
const [
|
|
5674
|
-
() => targetResource.
|
|
5675
|
-
[config.field]: consolidatedValue
|
|
5676
|
-
})
|
|
5819
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
5820
|
+
() => targetResource.get(originalId)
|
|
5677
5821
|
);
|
|
5822
|
+
let updateOk, updateErr, updateResult;
|
|
5823
|
+
if (!recordOk || !record) {
|
|
5824
|
+
if (config.verbose) {
|
|
5825
|
+
console.log(
|
|
5826
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist yet. Will attempt update anyway (expected to fail).`
|
|
5827
|
+
);
|
|
5828
|
+
}
|
|
5829
|
+
const minimalRecord = { id: originalId };
|
|
5830
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
5831
|
+
lodash.set(minimalRecord, fieldPath, value);
|
|
5832
|
+
}
|
|
5833
|
+
const result = await tryFn(
|
|
5834
|
+
() => targetResource.update(originalId, minimalRecord)
|
|
5835
|
+
);
|
|
5836
|
+
updateOk = result[0];
|
|
5837
|
+
updateErr = result[1];
|
|
5838
|
+
updateResult = result[2];
|
|
5839
|
+
} else {
|
|
5840
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
5841
|
+
lodash.set(record, fieldPath, value);
|
|
5842
|
+
}
|
|
5843
|
+
const result = await tryFn(
|
|
5844
|
+
() => targetResource.update(originalId, record)
|
|
5845
|
+
);
|
|
5846
|
+
updateOk = result[0];
|
|
5847
|
+
updateErr = result[1];
|
|
5848
|
+
updateResult = result[2];
|
|
5849
|
+
}
|
|
5850
|
+
const consolidatedValue = consolidatedValues[config.field] || (record ? lodash.get(record, config.field, 0) : 0);
|
|
5678
5851
|
if (config.verbose) {
|
|
5679
5852
|
console.log(
|
|
5680
5853
|
`\u{1F525} [DEBUG] AFTER targetResource.update() {
|
|
5681
5854
|
updateOk: ${updateOk},
|
|
5682
5855
|
updateErr: ${updateErr?.message || "undefined"},
|
|
5683
|
-
|
|
5684
|
-
hasField: ${updateResult?.[config.field]}
|
|
5856
|
+
consolidatedValue (main field): ${consolidatedValue}
|
|
5685
5857
|
}`
|
|
5686
5858
|
);
|
|
5687
5859
|
}
|
|
@@ -5689,24 +5861,27 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5689
5861
|
const [verifyOk, verifyErr, verifiedRecord] = await tryFn(
|
|
5690
5862
|
() => targetResource.get(originalId, { skipCache: true })
|
|
5691
5863
|
);
|
|
5692
|
-
|
|
5693
|
-
|
|
5694
|
-
|
|
5695
|
-
|
|
5696
|
-
|
|
5697
|
-
|
|
5864
|
+
for (const [fieldPath, expectedValue] of Object.entries(consolidatedValues)) {
|
|
5865
|
+
const actualValue = lodash.get(verifiedRecord, fieldPath);
|
|
5866
|
+
const match = actualValue === expectedValue;
|
|
5867
|
+
console.log(
|
|
5868
|
+
`\u{1F525} [DEBUG] VERIFICATION ${fieldPath} {
|
|
5869
|
+
expectedValue: ${expectedValue},
|
|
5870
|
+
actualValue: ${actualValue},
|
|
5871
|
+
${match ? "\u2705 MATCH" : "\u274C MISMATCH"}
|
|
5698
5872
|
}`
|
|
5699
|
-
|
|
5700
|
-
|
|
5701
|
-
|
|
5702
|
-
|
|
5873
|
+
);
|
|
5874
|
+
if (!match) {
|
|
5875
|
+
console.error(
|
|
5876
|
+
`\u274C [CRITICAL BUG] Update reported success but value not persisted!
|
|
5703
5877
|
Resource: ${config.resource}
|
|
5704
|
-
|
|
5878
|
+
FieldPath: ${fieldPath}
|
|
5705
5879
|
Record ID: ${originalId}
|
|
5706
|
-
Expected: ${
|
|
5707
|
-
Actually got: ${
|
|
5880
|
+
Expected: ${expectedValue}
|
|
5881
|
+
Actually got: ${actualValue}
|
|
5708
5882
|
This indicates a bug in s3db.js resource.update()`
|
|
5709
|
-
|
|
5883
|
+
);
|
|
5884
|
+
}
|
|
5710
5885
|
}
|
|
5711
5886
|
}
|
|
5712
5887
|
if (!updateOk) {
|
|
@@ -5891,6 +6066,38 @@ async function recalculateRecord(originalId, transactionResource, targetResource
|
|
|
5891
6066
|
`[EventualConsistency] ${config.resource}.${config.field} - Found ${allTransactions.length} total transactions for ${originalId}, marking all as pending...`
|
|
5892
6067
|
);
|
|
5893
6068
|
}
|
|
6069
|
+
const hasAnchor = allTransactions.some((txn) => txn.source === "anchor");
|
|
6070
|
+
if (!hasAnchor) {
|
|
6071
|
+
const now = /* @__PURE__ */ new Date();
|
|
6072
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6073
|
+
const oldestTransaction = allTransactions.sort(
|
|
6074
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
6075
|
+
)[0];
|
|
6076
|
+
const anchorTimestamp = oldestTransaction ? new Date(new Date(oldestTransaction.timestamp).getTime() - 1).toISOString() : now.toISOString();
|
|
6077
|
+
const anchorCohortInfo = getCohortInfo(new Date(anchorTimestamp), config.cohort.timezone, config.verbose);
|
|
6078
|
+
const anchorTransaction = {
|
|
6079
|
+
id: idGenerator(),
|
|
6080
|
+
originalId,
|
|
6081
|
+
field: config.field,
|
|
6082
|
+
fieldPath: config.field,
|
|
6083
|
+
value: 0,
|
|
6084
|
+
// Always 0 for recalculate - we start from scratch
|
|
6085
|
+
operation: "set",
|
|
6086
|
+
timestamp: anchorTimestamp,
|
|
6087
|
+
cohortDate: anchorCohortInfo.date,
|
|
6088
|
+
cohortHour: anchorCohortInfo.hour,
|
|
6089
|
+
cohortMonth: anchorCohortInfo.month,
|
|
6090
|
+
source: "anchor",
|
|
6091
|
+
applied: true
|
|
6092
|
+
// Anchor is always applied
|
|
6093
|
+
};
|
|
6094
|
+
await transactionResource.insert(anchorTransaction);
|
|
6095
|
+
if (config.verbose) {
|
|
6096
|
+
console.log(
|
|
6097
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Created anchor transaction for ${originalId} with value 0`
|
|
6098
|
+
);
|
|
6099
|
+
}
|
|
6100
|
+
}
|
|
5894
6101
|
const transactionsToReset = allTransactions.filter((txn) => txn.source !== "anchor");
|
|
5895
6102
|
const { results, errors } = await promisePool.PromisePool.for(transactionsToReset).withConcurrency(10).process(async (txn) => {
|
|
5896
6103
|
const [ok, err] = await tryFn(
|
|
@@ -6518,16 +6725,151 @@ async function getTopRecords(resourceName, field, options, fieldHandlers) {
|
|
|
6518
6725
|
});
|
|
6519
6726
|
return records.slice(0, limit);
|
|
6520
6727
|
}
|
|
6728
|
+
async function getYearByDay(resourceName, field, year, options, fieldHandlers) {
|
|
6729
|
+
const startDate = `${year}-01-01`;
|
|
6730
|
+
const endDate = `${year}-12-31`;
|
|
6731
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6732
|
+
period: "day",
|
|
6733
|
+
startDate,
|
|
6734
|
+
endDate
|
|
6735
|
+
}, fieldHandlers);
|
|
6736
|
+
if (options.fillGaps) {
|
|
6737
|
+
return fillGaps(data, "day", startDate, endDate);
|
|
6738
|
+
}
|
|
6739
|
+
return data;
|
|
6740
|
+
}
|
|
6741
|
+
async function getWeekByDay(resourceName, field, week, options, fieldHandlers) {
|
|
6742
|
+
const year = parseInt(week.substring(0, 4));
|
|
6743
|
+
const weekNum = parseInt(week.substring(6, 8));
|
|
6744
|
+
const jan4 = new Date(Date.UTC(year, 0, 4));
|
|
6745
|
+
const jan4Day = jan4.getUTCDay() || 7;
|
|
6746
|
+
const firstMonday = new Date(Date.UTC(year, 0, 4 - jan4Day + 1));
|
|
6747
|
+
const weekStart = new Date(firstMonday);
|
|
6748
|
+
weekStart.setUTCDate(weekStart.getUTCDate() + (weekNum - 1) * 7);
|
|
6749
|
+
const days = [];
|
|
6750
|
+
for (let i = 0; i < 7; i++) {
|
|
6751
|
+
const day = new Date(weekStart);
|
|
6752
|
+
day.setUTCDate(weekStart.getUTCDate() + i);
|
|
6753
|
+
days.push(day.toISOString().substring(0, 10));
|
|
6754
|
+
}
|
|
6755
|
+
const startDate = days[0];
|
|
6756
|
+
const endDate = days[6];
|
|
6757
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6758
|
+
period: "day",
|
|
6759
|
+
startDate,
|
|
6760
|
+
endDate
|
|
6761
|
+
}, fieldHandlers);
|
|
6762
|
+
if (options.fillGaps) {
|
|
6763
|
+
return fillGaps(data, "day", startDate, endDate);
|
|
6764
|
+
}
|
|
6765
|
+
return data;
|
|
6766
|
+
}
|
|
6767
|
+
async function getWeekByHour(resourceName, field, week, options, fieldHandlers) {
|
|
6768
|
+
const year = parseInt(week.substring(0, 4));
|
|
6769
|
+
const weekNum = parseInt(week.substring(6, 8));
|
|
6770
|
+
const jan4 = new Date(Date.UTC(year, 0, 4));
|
|
6771
|
+
const jan4Day = jan4.getUTCDay() || 7;
|
|
6772
|
+
const firstMonday = new Date(Date.UTC(year, 0, 4 - jan4Day + 1));
|
|
6773
|
+
const weekStart = new Date(firstMonday);
|
|
6774
|
+
weekStart.setUTCDate(weekStart.getUTCDate() + (weekNum - 1) * 7);
|
|
6775
|
+
const weekEnd = new Date(weekStart);
|
|
6776
|
+
weekEnd.setUTCDate(weekEnd.getUTCDate() + 6);
|
|
6777
|
+
const startDate = weekStart.toISOString().substring(0, 10);
|
|
6778
|
+
const endDate = weekEnd.toISOString().substring(0, 10);
|
|
6779
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6780
|
+
period: "hour",
|
|
6781
|
+
startDate,
|
|
6782
|
+
endDate
|
|
6783
|
+
}, fieldHandlers);
|
|
6784
|
+
if (options.fillGaps) {
|
|
6785
|
+
return fillGaps(data, "hour", startDate, endDate);
|
|
6786
|
+
}
|
|
6787
|
+
return data;
|
|
6788
|
+
}
|
|
6789
|
+
async function getLastNHours(resourceName, field, hours = 24, options, fieldHandlers) {
|
|
6790
|
+
const now = /* @__PURE__ */ new Date();
|
|
6791
|
+
const hoursAgo = new Date(now);
|
|
6792
|
+
hoursAgo.setHours(hoursAgo.getHours() - hours + 1);
|
|
6793
|
+
const startHour = hoursAgo.toISOString().substring(0, 13);
|
|
6794
|
+
const endHour = now.toISOString().substring(0, 13);
|
|
6795
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6796
|
+
period: "hour",
|
|
6797
|
+
startDate: startHour,
|
|
6798
|
+
endDate: endHour
|
|
6799
|
+
}, fieldHandlers);
|
|
6800
|
+
if (options.fillGaps) {
|
|
6801
|
+
const result = [];
|
|
6802
|
+
const emptyRecord = { count: 0, sum: 0, avg: 0, min: 0, max: 0, recordCount: 0 };
|
|
6803
|
+
const dataMap = new Map(data.map((d) => [d.cohort, d]));
|
|
6804
|
+
const current = new Date(hoursAgo);
|
|
6805
|
+
for (let i = 0; i < hours; i++) {
|
|
6806
|
+
const cohort = current.toISOString().substring(0, 13);
|
|
6807
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
6808
|
+
current.setHours(current.getHours() + 1);
|
|
6809
|
+
}
|
|
6810
|
+
return result;
|
|
6811
|
+
}
|
|
6812
|
+
return data;
|
|
6813
|
+
}
|
|
6814
|
+
async function getLastNWeeks(resourceName, field, weeks = 4, options, fieldHandlers) {
|
|
6815
|
+
const now = /* @__PURE__ */ new Date();
|
|
6816
|
+
const weeksAgo = new Date(now);
|
|
6817
|
+
weeksAgo.setDate(weeksAgo.getDate() - weeks * 7);
|
|
6818
|
+
const weekCohorts = [];
|
|
6819
|
+
const currentDate = new Date(weeksAgo);
|
|
6820
|
+
while (currentDate <= now) {
|
|
6821
|
+
const weekCohort = getCohortWeekFromDate(currentDate);
|
|
6822
|
+
if (!weekCohorts.includes(weekCohort)) {
|
|
6823
|
+
weekCohorts.push(weekCohort);
|
|
6824
|
+
}
|
|
6825
|
+
currentDate.setDate(currentDate.getDate() + 7);
|
|
6826
|
+
}
|
|
6827
|
+
const startWeek = weekCohorts[0];
|
|
6828
|
+
const endWeek = weekCohorts[weekCohorts.length - 1];
|
|
6829
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6830
|
+
period: "week",
|
|
6831
|
+
startDate: startWeek,
|
|
6832
|
+
endDate: endWeek
|
|
6833
|
+
}, fieldHandlers);
|
|
6834
|
+
return data;
|
|
6835
|
+
}
|
|
6836
|
+
async function getLastNMonths(resourceName, field, months = 12, options, fieldHandlers) {
|
|
6837
|
+
const now = /* @__PURE__ */ new Date();
|
|
6838
|
+
const monthsAgo = new Date(now);
|
|
6839
|
+
monthsAgo.setMonth(monthsAgo.getMonth() - months + 1);
|
|
6840
|
+
const startDate = monthsAgo.toISOString().substring(0, 7);
|
|
6841
|
+
const endDate = now.toISOString().substring(0, 7);
|
|
6842
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6843
|
+
period: "month",
|
|
6844
|
+
startDate,
|
|
6845
|
+
endDate
|
|
6846
|
+
}, fieldHandlers);
|
|
6847
|
+
if (options.fillGaps) {
|
|
6848
|
+
const result = [];
|
|
6849
|
+
const emptyRecord = { count: 0, sum: 0, avg: 0, min: 0, max: 0, recordCount: 0 };
|
|
6850
|
+
const dataMap = new Map(data.map((d) => [d.cohort, d]));
|
|
6851
|
+
const current = new Date(monthsAgo);
|
|
6852
|
+
for (let i = 0; i < months; i++) {
|
|
6853
|
+
const cohort = current.toISOString().substring(0, 7);
|
|
6854
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
6855
|
+
current.setMonth(current.getMonth() + 1);
|
|
6856
|
+
}
|
|
6857
|
+
return result;
|
|
6858
|
+
}
|
|
6859
|
+
return data;
|
|
6860
|
+
}
|
|
6521
6861
|
|
|
6522
6862
|
function addHelperMethods(resource, plugin, config) {
|
|
6523
6863
|
resource.set = async (id, field, value) => {
|
|
6524
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
6864
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
6525
6865
|
const now = /* @__PURE__ */ new Date();
|
|
6526
6866
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6527
6867
|
const transaction = {
|
|
6528
6868
|
id: idGenerator(),
|
|
6529
6869
|
originalId: id,
|
|
6530
6870
|
field: handler.field,
|
|
6871
|
+
fieldPath,
|
|
6872
|
+
// Store full path for nested access
|
|
6531
6873
|
value,
|
|
6532
6874
|
operation: "set",
|
|
6533
6875
|
timestamp: now.toISOString(),
|
|
@@ -6539,18 +6881,20 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6539
6881
|
};
|
|
6540
6882
|
await handler.transactionResource.insert(transaction);
|
|
6541
6883
|
if (config.mode === "sync") {
|
|
6542
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
6884
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6543
6885
|
}
|
|
6544
6886
|
return value;
|
|
6545
6887
|
};
|
|
6546
6888
|
resource.add = async (id, field, amount) => {
|
|
6547
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6889
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6548
6890
|
const now = /* @__PURE__ */ new Date();
|
|
6549
6891
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6550
6892
|
const transaction = {
|
|
6551
6893
|
id: idGenerator(),
|
|
6552
6894
|
originalId: id,
|
|
6553
6895
|
field: handler.field,
|
|
6896
|
+
fieldPath,
|
|
6897
|
+
// Store full path for nested access
|
|
6554
6898
|
value: amount,
|
|
6555
6899
|
operation: "add",
|
|
6556
6900
|
timestamp: now.toISOString(),
|
|
@@ -6562,20 +6906,24 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6562
6906
|
};
|
|
6563
6907
|
await handler.transactionResource.insert(transaction);
|
|
6564
6908
|
if (config.mode === "sync") {
|
|
6565
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
6909
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6566
6910
|
}
|
|
6567
6911
|
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
6568
|
-
|
|
6912
|
+
if (!ok || !record) return amount;
|
|
6913
|
+
const lodash = await import('lodash-es');
|
|
6914
|
+
const currentValue = lodash.get(record, fieldPath, 0);
|
|
6569
6915
|
return currentValue + amount;
|
|
6570
6916
|
};
|
|
6571
6917
|
resource.sub = async (id, field, amount) => {
|
|
6572
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6918
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6573
6919
|
const now = /* @__PURE__ */ new Date();
|
|
6574
6920
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6575
6921
|
const transaction = {
|
|
6576
6922
|
id: idGenerator(),
|
|
6577
6923
|
originalId: id,
|
|
6578
6924
|
field: handler.field,
|
|
6925
|
+
fieldPath,
|
|
6926
|
+
// Store full path for nested access
|
|
6579
6927
|
value: amount,
|
|
6580
6928
|
operation: "sub",
|
|
6581
6929
|
timestamp: now.toISOString(),
|
|
@@ -6587,10 +6935,12 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6587
6935
|
};
|
|
6588
6936
|
await handler.transactionResource.insert(transaction);
|
|
6589
6937
|
if (config.mode === "sync") {
|
|
6590
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
6938
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6591
6939
|
}
|
|
6592
6940
|
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
6593
|
-
|
|
6941
|
+
if (!ok || !record) return -amount;
|
|
6942
|
+
const lodash = await import('lodash-es');
|
|
6943
|
+
const currentValue = lodash.get(record, fieldPath, 0);
|
|
6594
6944
|
return currentValue - amount;
|
|
6595
6945
|
};
|
|
6596
6946
|
resource.consolidate = async (id, field) => {
|
|
@@ -6676,6 +7026,8 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6676
7026
|
id: "string|required",
|
|
6677
7027
|
originalId: "string|required",
|
|
6678
7028
|
field: "string|required",
|
|
7029
|
+
fieldPath: "string|optional",
|
|
7030
|
+
// Support for nested field paths (e.g., 'utmResults.medium')
|
|
6679
7031
|
value: "number|required",
|
|
6680
7032
|
operation: "string|required",
|
|
6681
7033
|
timestamp: "string|required",
|
|
@@ -7174,6 +7526,72 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
7174
7526
|
async getTopRecords(resourceName, field, options = {}) {
|
|
7175
7527
|
return await getTopRecords(resourceName, field, options, this.fieldHandlers);
|
|
7176
7528
|
}
|
|
7529
|
+
/**
|
|
7530
|
+
* Get analytics for entire year, broken down by days
|
|
7531
|
+
* @param {string} resourceName - Resource name
|
|
7532
|
+
* @param {string} field - Field name
|
|
7533
|
+
* @param {number} year - Year (e.g., 2025)
|
|
7534
|
+
* @param {Object} options - Options
|
|
7535
|
+
* @returns {Promise<Array>} Daily analytics for the year (up to 365/366 records)
|
|
7536
|
+
*/
|
|
7537
|
+
async getYearByDay(resourceName, field, year, options = {}) {
|
|
7538
|
+
return await getYearByDay(resourceName, field, year, options, this.fieldHandlers);
|
|
7539
|
+
}
|
|
7540
|
+
/**
|
|
7541
|
+
* Get analytics for entire week, broken down by days
|
|
7542
|
+
* @param {string} resourceName - Resource name
|
|
7543
|
+
* @param {string} field - Field name
|
|
7544
|
+
* @param {string} week - Week in YYYY-Www format (e.g., '2025-W42')
|
|
7545
|
+
* @param {Object} options - Options
|
|
7546
|
+
* @returns {Promise<Array>} Daily analytics for the week (7 records)
|
|
7547
|
+
*/
|
|
7548
|
+
async getWeekByDay(resourceName, field, week, options = {}) {
|
|
7549
|
+
return await getWeekByDay(resourceName, field, week, options, this.fieldHandlers);
|
|
7550
|
+
}
|
|
7551
|
+
/**
|
|
7552
|
+
* Get analytics for entire week, broken down by hours
|
|
7553
|
+
* @param {string} resourceName - Resource name
|
|
7554
|
+
* @param {string} field - Field name
|
|
7555
|
+
* @param {string} week - Week in YYYY-Www format (e.g., '2025-W42')
|
|
7556
|
+
* @param {Object} options - Options
|
|
7557
|
+
* @returns {Promise<Array>} Hourly analytics for the week (168 records)
|
|
7558
|
+
*/
|
|
7559
|
+
async getWeekByHour(resourceName, field, week, options = {}) {
|
|
7560
|
+
return await getWeekByHour(resourceName, field, week, options, this.fieldHandlers);
|
|
7561
|
+
}
|
|
7562
|
+
/**
|
|
7563
|
+
* Get analytics for last N hours
|
|
7564
|
+
* @param {string} resourceName - Resource name
|
|
7565
|
+
* @param {string} field - Field name
|
|
7566
|
+
* @param {number} hours - Number of hours to look back (default: 24)
|
|
7567
|
+
* @param {Object} options - Options
|
|
7568
|
+
* @returns {Promise<Array>} Hourly analytics
|
|
7569
|
+
*/
|
|
7570
|
+
async getLastNHours(resourceName, field, hours = 24, options = {}) {
|
|
7571
|
+
return await getLastNHours(resourceName, field, hours, options, this.fieldHandlers);
|
|
7572
|
+
}
|
|
7573
|
+
/**
|
|
7574
|
+
* Get analytics for last N weeks
|
|
7575
|
+
* @param {string} resourceName - Resource name
|
|
7576
|
+
* @param {string} field - Field name
|
|
7577
|
+
* @param {number} weeks - Number of weeks to look back (default: 4)
|
|
7578
|
+
* @param {Object} options - Options
|
|
7579
|
+
* @returns {Promise<Array>} Weekly analytics
|
|
7580
|
+
*/
|
|
7581
|
+
async getLastNWeeks(resourceName, field, weeks = 4, options = {}) {
|
|
7582
|
+
return await getLastNWeeks(resourceName, field, weeks, options, this.fieldHandlers);
|
|
7583
|
+
}
|
|
7584
|
+
/**
|
|
7585
|
+
* Get analytics for last N months
|
|
7586
|
+
* @param {string} resourceName - Resource name
|
|
7587
|
+
* @param {string} field - Field name
|
|
7588
|
+
* @param {number} months - Number of months to look back (default: 12)
|
|
7589
|
+
* @param {Object} options - Options
|
|
7590
|
+
* @returns {Promise<Array>} Monthly analytics
|
|
7591
|
+
*/
|
|
7592
|
+
async getLastNMonths(resourceName, field, months = 12, options = {}) {
|
|
7593
|
+
return await getLastNMonths(resourceName, field, months, options, this.fieldHandlers);
|
|
7594
|
+
}
|
|
7177
7595
|
}
|
|
7178
7596
|
|
|
7179
7597
|
class FullTextPlugin extends Plugin {
|
|
@@ -13192,7 +13610,7 @@ class Database extends EventEmitter {
|
|
|
13192
13610
|
this.id = idGenerator(7);
|
|
13193
13611
|
this.version = "1";
|
|
13194
13612
|
this.s3dbVersion = (() => {
|
|
13195
|
-
const [ok, err, version] = tryFn(() => true ? "11.0
|
|
13613
|
+
const [ok, err, version] = tryFn(() => true ? "11.1.0" : "latest");
|
|
13196
13614
|
return ok ? version : "latest";
|
|
13197
13615
|
})();
|
|
13198
13616
|
this.resources = {};
|