s3db.js 11.0.5 → 11.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -2
- package/SECURITY.md +76 -0
- package/dist/s3db.cjs.js +446 -86
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +446 -86
- package/dist/s3db.es.js.map +1 -1
- package/package.json +3 -1
- package/src/concerns/crypto.js +7 -14
- package/src/plugins/eventual-consistency/analytics.js +164 -2
- package/src/plugins/eventual-consistency/consolidation.js +228 -80
- package/src/plugins/eventual-consistency/helpers.js +24 -8
- package/src/plugins/eventual-consistency/install.js +2 -1
- package/src/plugins/eventual-consistency/utils.js +218 -4
- package/src/concerns/advanced-metadata-encoding.js +0 -440
package/dist/s3db.cjs.js
CHANGED
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
4
|
|
|
5
|
+
var crypto = require('crypto');
|
|
5
6
|
var nanoid = require('nanoid');
|
|
6
7
|
var EventEmitter = require('events');
|
|
7
8
|
var promises = require('fs/promises');
|
|
8
9
|
var fs = require('fs');
|
|
9
10
|
var promises$1 = require('stream/promises');
|
|
10
11
|
var path = require('path');
|
|
11
|
-
var crypto = require('crypto');
|
|
12
12
|
var zlib = require('node:zlib');
|
|
13
13
|
var os = require('os');
|
|
14
14
|
var jsonStableStringify = require('json-stable-stringify');
|
|
@@ -528,15 +528,7 @@ function tryFnSync(fn) {
|
|
|
528
528
|
async function dynamicCrypto() {
|
|
529
529
|
let lib;
|
|
530
530
|
if (typeof process !== "undefined") {
|
|
531
|
-
|
|
532
|
-
const { webcrypto } = await import('crypto');
|
|
533
|
-
return webcrypto;
|
|
534
|
-
});
|
|
535
|
-
if (ok) {
|
|
536
|
-
lib = result;
|
|
537
|
-
} else {
|
|
538
|
-
throw new CryptoError("Crypto API not available", { original: err, context: "dynamicCrypto" });
|
|
539
|
-
}
|
|
531
|
+
lib = crypto.webcrypto;
|
|
540
532
|
} else if (typeof window !== "undefined") {
|
|
541
533
|
lib = window.crypto;
|
|
542
534
|
}
|
|
@@ -590,8 +582,7 @@ async function md5(data) {
|
|
|
590
582
|
throw new CryptoError("MD5 hashing is only available in Node.js environment", { context: "md5" });
|
|
591
583
|
}
|
|
592
584
|
const [ok, err, result] = await tryFn(async () => {
|
|
593
|
-
|
|
594
|
-
return createHash("md5").update(data).digest("base64");
|
|
585
|
+
return crypto.createHash("md5").update(data).digest("base64");
|
|
595
586
|
});
|
|
596
587
|
if (!ok) {
|
|
597
588
|
throw new CryptoError("MD5 hashing failed", { original: err, data });
|
|
@@ -5276,10 +5267,121 @@ function createFieldHandler(resourceName, fieldName) {
|
|
|
5276
5267
|
deferredSetup: false
|
|
5277
5268
|
};
|
|
5278
5269
|
}
|
|
5270
|
+
function validateNestedPath(resource, fieldPath) {
|
|
5271
|
+
const parts = fieldPath.split(".");
|
|
5272
|
+
const rootField = parts[0];
|
|
5273
|
+
if (!resource.attributes || !resource.attributes[rootField]) {
|
|
5274
|
+
return {
|
|
5275
|
+
valid: false,
|
|
5276
|
+
rootField,
|
|
5277
|
+
fullPath: fieldPath,
|
|
5278
|
+
error: `Root field "${rootField}" not found in resource attributes`
|
|
5279
|
+
};
|
|
5280
|
+
}
|
|
5281
|
+
if (parts.length === 1) {
|
|
5282
|
+
return { valid: true, rootField, fullPath: fieldPath };
|
|
5283
|
+
}
|
|
5284
|
+
let current = resource.attributes[rootField];
|
|
5285
|
+
let foundJson = false;
|
|
5286
|
+
let levelsAfterJson = 0;
|
|
5287
|
+
for (let i = 1; i < parts.length; i++) {
|
|
5288
|
+
const part = parts[i];
|
|
5289
|
+
if (foundJson) {
|
|
5290
|
+
levelsAfterJson++;
|
|
5291
|
+
if (levelsAfterJson > 1) {
|
|
5292
|
+
return {
|
|
5293
|
+
valid: false,
|
|
5294
|
+
rootField,
|
|
5295
|
+
fullPath: fieldPath,
|
|
5296
|
+
error: `Path "${fieldPath}" exceeds 1 level after 'json' field. Maximum nesting after 'json' is 1 level.`
|
|
5297
|
+
};
|
|
5298
|
+
}
|
|
5299
|
+
continue;
|
|
5300
|
+
}
|
|
5301
|
+
if (typeof current === "string") {
|
|
5302
|
+
if (current === "json" || current.startsWith("json|")) {
|
|
5303
|
+
foundJson = true;
|
|
5304
|
+
levelsAfterJson++;
|
|
5305
|
+
if (levelsAfterJson > 1) {
|
|
5306
|
+
return {
|
|
5307
|
+
valid: false,
|
|
5308
|
+
rootField,
|
|
5309
|
+
fullPath: fieldPath,
|
|
5310
|
+
error: `Path "${fieldPath}" exceeds 1 level after 'json' field`
|
|
5311
|
+
};
|
|
5312
|
+
}
|
|
5313
|
+
continue;
|
|
5314
|
+
}
|
|
5315
|
+
return {
|
|
5316
|
+
valid: false,
|
|
5317
|
+
rootField,
|
|
5318
|
+
fullPath: fieldPath,
|
|
5319
|
+
error: `Field "${parts.slice(0, i).join(".")}" is type "${current}" and cannot be nested`
|
|
5320
|
+
};
|
|
5321
|
+
}
|
|
5322
|
+
if (typeof current === "object") {
|
|
5323
|
+
if (current.$$type) {
|
|
5324
|
+
const type = current.$$type;
|
|
5325
|
+
if (type === "json" || type.includes("json")) {
|
|
5326
|
+
foundJson = true;
|
|
5327
|
+
levelsAfterJson++;
|
|
5328
|
+
continue;
|
|
5329
|
+
}
|
|
5330
|
+
if (type !== "object" && !type.includes("object")) {
|
|
5331
|
+
return {
|
|
5332
|
+
valid: false,
|
|
5333
|
+
rootField,
|
|
5334
|
+
fullPath: fieldPath,
|
|
5335
|
+
error: `Field "${parts.slice(0, i).join(".")}" is type "${type}" and cannot be nested`
|
|
5336
|
+
};
|
|
5337
|
+
}
|
|
5338
|
+
}
|
|
5339
|
+
if (!current[part]) {
|
|
5340
|
+
return {
|
|
5341
|
+
valid: false,
|
|
5342
|
+
rootField,
|
|
5343
|
+
fullPath: fieldPath,
|
|
5344
|
+
error: `Field "${part}" not found in "${parts.slice(0, i).join(".")}"`
|
|
5345
|
+
};
|
|
5346
|
+
}
|
|
5347
|
+
current = current[part];
|
|
5348
|
+
} else {
|
|
5349
|
+
return {
|
|
5350
|
+
valid: false,
|
|
5351
|
+
rootField,
|
|
5352
|
+
fullPath: fieldPath,
|
|
5353
|
+
error: `Invalid structure at "${parts.slice(0, i).join(".")}"`
|
|
5354
|
+
};
|
|
5355
|
+
}
|
|
5356
|
+
}
|
|
5357
|
+
return { valid: true, rootField, fullPath: fieldPath };
|
|
5358
|
+
}
|
|
5279
5359
|
function resolveFieldAndPlugin(resource, field, value) {
|
|
5280
5360
|
if (!resource._eventualConsistencyPlugins) {
|
|
5281
5361
|
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
5282
5362
|
}
|
|
5363
|
+
if (field.includes(".")) {
|
|
5364
|
+
const validation = validateNestedPath(resource, field);
|
|
5365
|
+
if (!validation.valid) {
|
|
5366
|
+
throw new Error(validation.error);
|
|
5367
|
+
}
|
|
5368
|
+
const rootField = validation.rootField;
|
|
5369
|
+
const fieldPlugin2 = resource._eventualConsistencyPlugins[rootField];
|
|
5370
|
+
if (!fieldPlugin2) {
|
|
5371
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5372
|
+
throw new Error(
|
|
5373
|
+
`No eventual consistency plugin found for root field "${rootField}". Available fields: ${availableFields}`
|
|
5374
|
+
);
|
|
5375
|
+
}
|
|
5376
|
+
return {
|
|
5377
|
+
field: rootField,
|
|
5378
|
+
// Root field for plugin lookup
|
|
5379
|
+
fieldPath: field,
|
|
5380
|
+
// Full path for nested access
|
|
5381
|
+
value,
|
|
5382
|
+
plugin: fieldPlugin2
|
|
5383
|
+
};
|
|
5384
|
+
}
|
|
5283
5385
|
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
5284
5386
|
if (!fieldPlugin) {
|
|
5285
5387
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
@@ -5287,7 +5389,7 @@ function resolveFieldAndPlugin(resource, field, value) {
|
|
|
5287
5389
|
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5288
5390
|
);
|
|
5289
5391
|
}
|
|
5290
|
-
return { field, value, plugin: fieldPlugin };
|
|
5392
|
+
return { field, fieldPath: field, value, plugin: fieldPlugin };
|
|
5291
5393
|
}
|
|
5292
5394
|
function groupByCohort(transactions, cohortField) {
|
|
5293
5395
|
const groups = {};
|
|
@@ -5301,6 +5403,38 @@ function groupByCohort(transactions, cohortField) {
|
|
|
5301
5403
|
}
|
|
5302
5404
|
return groups;
|
|
5303
5405
|
}
|
|
5406
|
+
function ensureCohortHour(transaction, timezone = "UTC", verbose = false) {
|
|
5407
|
+
if (transaction.cohortHour) {
|
|
5408
|
+
return transaction;
|
|
5409
|
+
}
|
|
5410
|
+
if (transaction.timestamp) {
|
|
5411
|
+
const date = new Date(transaction.timestamp);
|
|
5412
|
+
const cohortInfo = getCohortInfo(date, timezone, verbose);
|
|
5413
|
+
if (verbose) {
|
|
5414
|
+
console.log(
|
|
5415
|
+
`[EventualConsistency] Transaction ${transaction.id} missing cohortHour, calculated from timestamp: ${cohortInfo.hour}`
|
|
5416
|
+
);
|
|
5417
|
+
}
|
|
5418
|
+
transaction.cohortHour = cohortInfo.hour;
|
|
5419
|
+
if (!transaction.cohortWeek) {
|
|
5420
|
+
transaction.cohortWeek = cohortInfo.week;
|
|
5421
|
+
}
|
|
5422
|
+
if (!transaction.cohortMonth) {
|
|
5423
|
+
transaction.cohortMonth = cohortInfo.month;
|
|
5424
|
+
}
|
|
5425
|
+
} else if (verbose) {
|
|
5426
|
+
console.warn(
|
|
5427
|
+
`[EventualConsistency] Transaction ${transaction.id} missing both cohortHour and timestamp, cannot calculate cohort`
|
|
5428
|
+
);
|
|
5429
|
+
}
|
|
5430
|
+
return transaction;
|
|
5431
|
+
}
|
|
5432
|
+
function ensureCohortHours(transactions, timezone = "UTC", verbose = false) {
|
|
5433
|
+
if (!transactions || !Array.isArray(transactions)) {
|
|
5434
|
+
return transactions;
|
|
5435
|
+
}
|
|
5436
|
+
return transactions.map((txn) => ensureCohortHour(txn, timezone, verbose));
|
|
5437
|
+
}
|
|
5304
5438
|
|
|
5305
5439
|
function createPartitionConfig() {
|
|
5306
5440
|
const partitions = {
|
|
@@ -5532,10 +5666,10 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5532
5666
|
})
|
|
5533
5667
|
);
|
|
5534
5668
|
if (!ok || !transactions || transactions.length === 0) {
|
|
5535
|
-
const [
|
|
5669
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
5536
5670
|
() => targetResource.get(originalId)
|
|
5537
5671
|
);
|
|
5538
|
-
const currentValue2 =
|
|
5672
|
+
const currentValue2 = recordOk2 && record2 ? record2[config.field] || 0 : 0;
|
|
5539
5673
|
if (config.verbose) {
|
|
5540
5674
|
console.log(
|
|
5541
5675
|
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions for ${originalId}, skipping`
|
|
@@ -5570,6 +5704,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5570
5704
|
);
|
|
5571
5705
|
}
|
|
5572
5706
|
currentValue = 0;
|
|
5707
|
+
appliedTransactions.length = 0;
|
|
5573
5708
|
} else {
|
|
5574
5709
|
appliedTransactions.sort(
|
|
5575
5710
|
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
@@ -5577,42 +5712,46 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5577
5712
|
const hasSetInApplied = appliedTransactions.some((t) => t.operation === "set");
|
|
5578
5713
|
if (!hasSetInApplied) {
|
|
5579
5714
|
const recordValue = recordExists[config.field] || 0;
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
|
|
5593
|
-
|
|
5594
|
-
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5715
|
+
if (typeof recordValue === "number") {
|
|
5716
|
+
let appliedDelta = 0;
|
|
5717
|
+
for (const t of appliedTransactions) {
|
|
5718
|
+
if (t.operation === "add") appliedDelta += t.value;
|
|
5719
|
+
else if (t.operation === "sub") appliedDelta -= t.value;
|
|
5720
|
+
}
|
|
5721
|
+
const baseValue = recordValue - appliedDelta;
|
|
5722
|
+
const hasExistingAnchor = appliedTransactions.some((t) => t.source === "anchor");
|
|
5723
|
+
if (baseValue !== 0 && typeof baseValue === "number" && !hasExistingAnchor) {
|
|
5724
|
+
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
5725
|
+
const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
|
|
5726
|
+
const anchorTransaction = {
|
|
5727
|
+
id: idGenerator(),
|
|
5728
|
+
originalId,
|
|
5729
|
+
field: config.field,
|
|
5730
|
+
fieldPath: config.field,
|
|
5731
|
+
// Add fieldPath for consistency
|
|
5732
|
+
value: baseValue,
|
|
5733
|
+
operation: "set",
|
|
5734
|
+
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(),
|
|
5735
|
+
// 1ms before first txn to ensure it's first
|
|
5736
|
+
cohortDate: cohortInfo.date,
|
|
5737
|
+
cohortHour: cohortInfo.hour,
|
|
5738
|
+
cohortMonth: cohortInfo.month,
|
|
5739
|
+
source: "anchor",
|
|
5740
|
+
applied: true
|
|
5741
|
+
};
|
|
5742
|
+
await transactionResource.insert(anchorTransaction);
|
|
5743
|
+
appliedTransactions.unshift(anchorTransaction);
|
|
5744
|
+
}
|
|
5606
5745
|
}
|
|
5607
5746
|
}
|
|
5608
5747
|
currentValue = config.reducer(appliedTransactions);
|
|
5609
5748
|
}
|
|
5610
5749
|
} else {
|
|
5611
|
-
const [
|
|
5750
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
5612
5751
|
() => targetResource.get(originalId)
|
|
5613
5752
|
);
|
|
5614
|
-
currentValue =
|
|
5615
|
-
if (currentValue !== 0) {
|
|
5753
|
+
currentValue = recordOk2 && record2 ? record2[config.field] || 0 : 0;
|
|
5754
|
+
if (currentValue !== 0 && typeof currentValue === "number") {
|
|
5616
5755
|
let anchorTimestamp;
|
|
5617
5756
|
if (transactions && transactions.length > 0) {
|
|
5618
5757
|
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
@@ -5625,6 +5764,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5625
5764
|
id: idGenerator(),
|
|
5626
5765
|
originalId,
|
|
5627
5766
|
field: config.field,
|
|
5767
|
+
fieldPath: config.field,
|
|
5768
|
+
// Add fieldPath for consistency
|
|
5628
5769
|
value: currentValue,
|
|
5629
5770
|
operation: "set",
|
|
5630
5771
|
timestamp: anchorTimestamp,
|
|
@@ -5650,38 +5791,101 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5650
5791
|
transactions.sort(
|
|
5651
5792
|
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5652
5793
|
);
|
|
5653
|
-
const
|
|
5654
|
-
|
|
5655
|
-
|
|
5794
|
+
const transactionsByPath = {};
|
|
5795
|
+
for (const txn of transactions) {
|
|
5796
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
5797
|
+
if (!transactionsByPath[path]) {
|
|
5798
|
+
transactionsByPath[path] = [];
|
|
5799
|
+
}
|
|
5800
|
+
transactionsByPath[path].push(txn);
|
|
5656
5801
|
}
|
|
5657
|
-
const
|
|
5658
|
-
if (
|
|
5659
|
-
|
|
5660
|
-
|
|
5661
|
-
|
|
5802
|
+
const appliedByPath = {};
|
|
5803
|
+
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
5804
|
+
for (const txn of appliedTransactions) {
|
|
5805
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
5806
|
+
if (!appliedByPath[path]) {
|
|
5807
|
+
appliedByPath[path] = [];
|
|
5808
|
+
}
|
|
5809
|
+
appliedByPath[path].push(txn);
|
|
5810
|
+
}
|
|
5811
|
+
}
|
|
5812
|
+
const consolidatedValues = {};
|
|
5813
|
+
const lodash = await import('lodash-es');
|
|
5814
|
+
const [currentRecordOk, currentRecordErr, currentRecord] = await tryFn(
|
|
5815
|
+
() => targetResource.get(originalId)
|
|
5816
|
+
);
|
|
5817
|
+
for (const [fieldPath, pathTransactions] of Object.entries(transactionsByPath)) {
|
|
5818
|
+
let pathCurrentValue = 0;
|
|
5819
|
+
if (appliedByPath[fieldPath] && appliedByPath[fieldPath].length > 0) {
|
|
5820
|
+
appliedByPath[fieldPath].sort(
|
|
5821
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5822
|
+
);
|
|
5823
|
+
pathCurrentValue = config.reducer(appliedByPath[fieldPath]);
|
|
5824
|
+
} else {
|
|
5825
|
+
if (currentRecordOk && currentRecord) {
|
|
5826
|
+
const recordValue = lodash.get(currentRecord, fieldPath, 0);
|
|
5827
|
+
if (typeof recordValue === "number") {
|
|
5828
|
+
pathCurrentValue = recordValue;
|
|
5829
|
+
}
|
|
5830
|
+
}
|
|
5831
|
+
}
|
|
5832
|
+
if (pathCurrentValue !== 0) {
|
|
5833
|
+
pathTransactions.unshift(createSyntheticSetTransaction(pathCurrentValue));
|
|
5834
|
+
}
|
|
5835
|
+
const pathConsolidatedValue = config.reducer(pathTransactions);
|
|
5836
|
+
consolidatedValues[fieldPath] = pathConsolidatedValue;
|
|
5837
|
+
if (config.verbose) {
|
|
5838
|
+
console.log(
|
|
5839
|
+
`[EventualConsistency] ${config.resource}.${fieldPath} - ${originalId}: ${pathCurrentValue} \u2192 ${pathConsolidatedValue} (${pathTransactions.length - (pathCurrentValue !== 0 ? 1 : 0)} pending txns)`
|
|
5840
|
+
);
|
|
5841
|
+
}
|
|
5662
5842
|
}
|
|
5663
5843
|
if (config.verbose) {
|
|
5664
5844
|
console.log(
|
|
5665
5845
|
`\u{1F525} [DEBUG] BEFORE targetResource.update() {
|
|
5666
5846
|
originalId: '${originalId}',
|
|
5667
|
-
|
|
5668
|
-
consolidatedValue: ${consolidatedValue},
|
|
5669
|
-
currentValue: ${currentValue}
|
|
5847
|
+
consolidatedValues: ${JSON.stringify(consolidatedValues, null, 2)}
|
|
5670
5848
|
}`
|
|
5671
5849
|
);
|
|
5672
5850
|
}
|
|
5673
|
-
const [
|
|
5674
|
-
() => targetResource.
|
|
5675
|
-
[config.field]: consolidatedValue
|
|
5676
|
-
})
|
|
5851
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
5852
|
+
() => targetResource.get(originalId)
|
|
5677
5853
|
);
|
|
5854
|
+
let updateOk, updateErr, updateResult;
|
|
5855
|
+
if (!recordOk || !record) {
|
|
5856
|
+
if (config.verbose) {
|
|
5857
|
+
console.log(
|
|
5858
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist yet. Will attempt update anyway (expected to fail).`
|
|
5859
|
+
);
|
|
5860
|
+
}
|
|
5861
|
+
const minimalRecord = { id: originalId };
|
|
5862
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
5863
|
+
lodash.set(minimalRecord, fieldPath, value);
|
|
5864
|
+
}
|
|
5865
|
+
const result = await tryFn(
|
|
5866
|
+
() => targetResource.update(originalId, minimalRecord)
|
|
5867
|
+
);
|
|
5868
|
+
updateOk = result[0];
|
|
5869
|
+
updateErr = result[1];
|
|
5870
|
+
updateResult = result[2];
|
|
5871
|
+
} else {
|
|
5872
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
5873
|
+
lodash.set(record, fieldPath, value);
|
|
5874
|
+
}
|
|
5875
|
+
const result = await tryFn(
|
|
5876
|
+
() => targetResource.update(originalId, record)
|
|
5877
|
+
);
|
|
5878
|
+
updateOk = result[0];
|
|
5879
|
+
updateErr = result[1];
|
|
5880
|
+
updateResult = result[2];
|
|
5881
|
+
}
|
|
5882
|
+
const consolidatedValue = consolidatedValues[config.field] || (record ? lodash.get(record, config.field, 0) : 0);
|
|
5678
5883
|
if (config.verbose) {
|
|
5679
5884
|
console.log(
|
|
5680
5885
|
`\u{1F525} [DEBUG] AFTER targetResource.update() {
|
|
5681
5886
|
updateOk: ${updateOk},
|
|
5682
5887
|
updateErr: ${updateErr?.message || "undefined"},
|
|
5683
|
-
|
|
5684
|
-
hasField: ${updateResult?.[config.field]}
|
|
5888
|
+
consolidatedValue (main field): ${consolidatedValue}
|
|
5685
5889
|
}`
|
|
5686
5890
|
);
|
|
5687
5891
|
}
|
|
@@ -5689,24 +5893,27 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5689
5893
|
const [verifyOk, verifyErr, verifiedRecord] = await tryFn(
|
|
5690
5894
|
() => targetResource.get(originalId, { skipCache: true })
|
|
5691
5895
|
);
|
|
5692
|
-
|
|
5693
|
-
|
|
5694
|
-
|
|
5695
|
-
|
|
5696
|
-
|
|
5697
|
-
|
|
5896
|
+
for (const [fieldPath, expectedValue] of Object.entries(consolidatedValues)) {
|
|
5897
|
+
const actualValue = lodash.get(verifiedRecord, fieldPath);
|
|
5898
|
+
const match = actualValue === expectedValue;
|
|
5899
|
+
console.log(
|
|
5900
|
+
`\u{1F525} [DEBUG] VERIFICATION ${fieldPath} {
|
|
5901
|
+
expectedValue: ${expectedValue},
|
|
5902
|
+
actualValue: ${actualValue},
|
|
5903
|
+
${match ? "\u2705 MATCH" : "\u274C MISMATCH"}
|
|
5698
5904
|
}`
|
|
5699
|
-
|
|
5700
|
-
|
|
5701
|
-
|
|
5702
|
-
|
|
5905
|
+
);
|
|
5906
|
+
if (!match) {
|
|
5907
|
+
console.error(
|
|
5908
|
+
`\u274C [CRITICAL BUG] Update reported success but value not persisted!
|
|
5703
5909
|
Resource: ${config.resource}
|
|
5704
|
-
|
|
5910
|
+
FieldPath: ${fieldPath}
|
|
5705
5911
|
Record ID: ${originalId}
|
|
5706
|
-
Expected: ${
|
|
5707
|
-
Actually got: ${
|
|
5912
|
+
Expected: ${expectedValue}
|
|
5913
|
+
Actually got: ${actualValue}
|
|
5708
5914
|
This indicates a bug in s3db.js resource.update()`
|
|
5709
|
-
|
|
5915
|
+
);
|
|
5916
|
+
}
|
|
5710
5917
|
}
|
|
5711
5918
|
}
|
|
5712
5919
|
if (!updateOk) {
|
|
@@ -5891,6 +6098,38 @@ async function recalculateRecord(originalId, transactionResource, targetResource
|
|
|
5891
6098
|
`[EventualConsistency] ${config.resource}.${config.field} - Found ${allTransactions.length} total transactions for ${originalId}, marking all as pending...`
|
|
5892
6099
|
);
|
|
5893
6100
|
}
|
|
6101
|
+
const hasAnchor = allTransactions.some((txn) => txn.source === "anchor");
|
|
6102
|
+
if (!hasAnchor) {
|
|
6103
|
+
const now = /* @__PURE__ */ new Date();
|
|
6104
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6105
|
+
const oldestTransaction = allTransactions.sort(
|
|
6106
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
6107
|
+
)[0];
|
|
6108
|
+
const anchorTimestamp = oldestTransaction ? new Date(new Date(oldestTransaction.timestamp).getTime() - 1).toISOString() : now.toISOString();
|
|
6109
|
+
const anchorCohortInfo = getCohortInfo(new Date(anchorTimestamp), config.cohort.timezone, config.verbose);
|
|
6110
|
+
const anchorTransaction = {
|
|
6111
|
+
id: idGenerator(),
|
|
6112
|
+
originalId,
|
|
6113
|
+
field: config.field,
|
|
6114
|
+
fieldPath: config.field,
|
|
6115
|
+
value: 0,
|
|
6116
|
+
// Always 0 for recalculate - we start from scratch
|
|
6117
|
+
operation: "set",
|
|
6118
|
+
timestamp: anchorTimestamp,
|
|
6119
|
+
cohortDate: anchorCohortInfo.date,
|
|
6120
|
+
cohortHour: anchorCohortInfo.hour,
|
|
6121
|
+
cohortMonth: anchorCohortInfo.month,
|
|
6122
|
+
source: "anchor",
|
|
6123
|
+
applied: true
|
|
6124
|
+
// Anchor is always applied
|
|
6125
|
+
};
|
|
6126
|
+
await transactionResource.insert(anchorTransaction);
|
|
6127
|
+
if (config.verbose) {
|
|
6128
|
+
console.log(
|
|
6129
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Created anchor transaction for ${originalId} with value 0`
|
|
6130
|
+
);
|
|
6131
|
+
}
|
|
6132
|
+
}
|
|
5894
6133
|
const transactionsToReset = allTransactions.filter((txn) => txn.source !== "anchor");
|
|
5895
6134
|
const { results, errors } = await promisePool.PromisePool.for(transactionsToReset).withConcurrency(10).process(async (txn) => {
|
|
5896
6135
|
const [ok, err] = await tryFn(
|
|
@@ -6320,7 +6559,10 @@ async function getAnalytics(resourceName, field, options, fieldHandlers) {
|
|
|
6320
6559
|
if (!handler.analyticsResource) {
|
|
6321
6560
|
throw new Error("Analytics not enabled for this plugin");
|
|
6322
6561
|
}
|
|
6323
|
-
const { period = "day", date, startDate, endDate, month, year, breakdown = false } = options;
|
|
6562
|
+
const { period = "day", date, startDate, endDate, month, year, breakdown = false, recordId } = options;
|
|
6563
|
+
if (recordId) {
|
|
6564
|
+
return await getAnalyticsForRecord(resourceName, field, recordId, options, handler);
|
|
6565
|
+
}
|
|
6324
6566
|
const [ok, err, allAnalytics] = await tryFn(
|
|
6325
6567
|
() => handler.analyticsResource.list()
|
|
6326
6568
|
);
|
|
@@ -6359,6 +6601,105 @@ async function getAnalytics(resourceName, field, options, fieldHandlers) {
|
|
|
6359
6601
|
recordCount: a.recordCount
|
|
6360
6602
|
}));
|
|
6361
6603
|
}
|
|
6604
|
+
async function getAnalyticsForRecord(resourceName, field, recordId, options, handler) {
|
|
6605
|
+
const { period = "day", date, startDate, endDate, month, year } = options;
|
|
6606
|
+
const [okTrue, errTrue, appliedTransactions] = await tryFn(
|
|
6607
|
+
() => handler.transactionResource.query({
|
|
6608
|
+
originalId: recordId,
|
|
6609
|
+
applied: true
|
|
6610
|
+
})
|
|
6611
|
+
);
|
|
6612
|
+
const [okFalse, errFalse, pendingTransactions] = await tryFn(
|
|
6613
|
+
() => handler.transactionResource.query({
|
|
6614
|
+
originalId: recordId,
|
|
6615
|
+
applied: false
|
|
6616
|
+
})
|
|
6617
|
+
);
|
|
6618
|
+
let allTransactions = [
|
|
6619
|
+
...okTrue && appliedTransactions ? appliedTransactions : [],
|
|
6620
|
+
...okFalse && pendingTransactions ? pendingTransactions : []
|
|
6621
|
+
];
|
|
6622
|
+
if (allTransactions.length === 0) {
|
|
6623
|
+
return [];
|
|
6624
|
+
}
|
|
6625
|
+
allTransactions = ensureCohortHours(allTransactions, handler.config?.cohort?.timezone || "UTC", false);
|
|
6626
|
+
let filtered = allTransactions;
|
|
6627
|
+
if (date) {
|
|
6628
|
+
if (period === "hour") {
|
|
6629
|
+
filtered = filtered.filter((t) => t.cohortHour && t.cohortHour.startsWith(date));
|
|
6630
|
+
} else if (period === "day") {
|
|
6631
|
+
filtered = filtered.filter((t) => t.cohortDate === date);
|
|
6632
|
+
} else if (period === "month") {
|
|
6633
|
+
filtered = filtered.filter((t) => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
6634
|
+
}
|
|
6635
|
+
} else if (startDate && endDate) {
|
|
6636
|
+
if (period === "hour") {
|
|
6637
|
+
filtered = filtered.filter((t) => t.cohortHour && t.cohortHour >= startDate && t.cohortHour <= endDate);
|
|
6638
|
+
} else if (period === "day") {
|
|
6639
|
+
filtered = filtered.filter((t) => t.cohortDate && t.cohortDate >= startDate && t.cohortDate <= endDate);
|
|
6640
|
+
} else if (period === "month") {
|
|
6641
|
+
filtered = filtered.filter((t) => t.cohortMonth && t.cohortMonth >= startDate && t.cohortMonth <= endDate);
|
|
6642
|
+
}
|
|
6643
|
+
} else if (month) {
|
|
6644
|
+
if (period === "hour") {
|
|
6645
|
+
filtered = filtered.filter((t) => t.cohortHour && t.cohortHour.startsWith(month));
|
|
6646
|
+
} else if (period === "day") {
|
|
6647
|
+
filtered = filtered.filter((t) => t.cohortDate && t.cohortDate.startsWith(month));
|
|
6648
|
+
}
|
|
6649
|
+
} else if (year) {
|
|
6650
|
+
if (period === "hour") {
|
|
6651
|
+
filtered = filtered.filter((t) => t.cohortHour && t.cohortHour.startsWith(String(year)));
|
|
6652
|
+
} else if (period === "day") {
|
|
6653
|
+
filtered = filtered.filter((t) => t.cohortDate && t.cohortDate.startsWith(String(year)));
|
|
6654
|
+
} else if (period === "month") {
|
|
6655
|
+
filtered = filtered.filter((t) => t.cohortMonth && t.cohortMonth.startsWith(String(year)));
|
|
6656
|
+
}
|
|
6657
|
+
}
|
|
6658
|
+
const cohortField = period === "hour" ? "cohortHour" : period === "day" ? "cohortDate" : "cohortMonth";
|
|
6659
|
+
const aggregated = aggregateTransactionsByCohort(filtered, cohortField);
|
|
6660
|
+
return aggregated;
|
|
6661
|
+
}
|
|
6662
|
+
function aggregateTransactionsByCohort(transactions, cohortField) {
|
|
6663
|
+
const groups = {};
|
|
6664
|
+
for (const txn of transactions) {
|
|
6665
|
+
const cohort = txn[cohortField];
|
|
6666
|
+
if (!cohort) continue;
|
|
6667
|
+
if (!groups[cohort]) {
|
|
6668
|
+
groups[cohort] = {
|
|
6669
|
+
cohort,
|
|
6670
|
+
count: 0,
|
|
6671
|
+
sum: 0,
|
|
6672
|
+
min: Infinity,
|
|
6673
|
+
max: -Infinity,
|
|
6674
|
+
recordCount: /* @__PURE__ */ new Set(),
|
|
6675
|
+
operations: {}
|
|
6676
|
+
};
|
|
6677
|
+
}
|
|
6678
|
+
const group = groups[cohort];
|
|
6679
|
+
const signedValue = txn.operation === "sub" ? -txn.value : txn.value;
|
|
6680
|
+
group.count++;
|
|
6681
|
+
group.sum += signedValue;
|
|
6682
|
+
group.min = Math.min(group.min, signedValue);
|
|
6683
|
+
group.max = Math.max(group.max, signedValue);
|
|
6684
|
+
group.recordCount.add(txn.originalId);
|
|
6685
|
+
const op = txn.operation;
|
|
6686
|
+
if (!group.operations[op]) {
|
|
6687
|
+
group.operations[op] = { count: 0, sum: 0 };
|
|
6688
|
+
}
|
|
6689
|
+
group.operations[op].count++;
|
|
6690
|
+
group.operations[op].sum += signedValue;
|
|
6691
|
+
}
|
|
6692
|
+
return Object.values(groups).map((g) => ({
|
|
6693
|
+
cohort: g.cohort,
|
|
6694
|
+
count: g.count,
|
|
6695
|
+
sum: g.sum,
|
|
6696
|
+
avg: g.sum / g.count,
|
|
6697
|
+
min: g.min === Infinity ? 0 : g.min,
|
|
6698
|
+
max: g.max === -Infinity ? 0 : g.max,
|
|
6699
|
+
recordCount: g.recordCount.size,
|
|
6700
|
+
operations: g.operations
|
|
6701
|
+
})).sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
6702
|
+
}
|
|
6362
6703
|
async function getMonthByDay(resourceName, field, month, options, fieldHandlers) {
|
|
6363
6704
|
const year = parseInt(month.substring(0, 4));
|
|
6364
6705
|
const monthNum = parseInt(month.substring(5, 7));
|
|
@@ -6393,6 +6734,8 @@ async function getLastNDays(resourceName, field, days, options, fieldHandlers) {
|
|
|
6393
6734
|
return date.toISOString().substring(0, 10);
|
|
6394
6735
|
}).reverse();
|
|
6395
6736
|
const data = await getAnalytics(resourceName, field, {
|
|
6737
|
+
...options,
|
|
6738
|
+
// ✅ Include all options (recordId, etc.)
|
|
6396
6739
|
period: "day",
|
|
6397
6740
|
startDate: dates[0],
|
|
6398
6741
|
endDate: dates[dates.length - 1]
|
|
@@ -6586,6 +6929,8 @@ async function getLastNHours(resourceName, field, hours = 24, options, fieldHand
|
|
|
6586
6929
|
const startHour = hoursAgo.toISOString().substring(0, 13);
|
|
6587
6930
|
const endHour = now.toISOString().substring(0, 13);
|
|
6588
6931
|
const data = await getAnalytics(resourceName, field, {
|
|
6932
|
+
...options,
|
|
6933
|
+
// ✅ Include all options (recordId, etc.)
|
|
6589
6934
|
period: "hour",
|
|
6590
6935
|
startDate: startHour,
|
|
6591
6936
|
endDate: endHour
|
|
@@ -6633,6 +6978,8 @@ async function getLastNMonths(resourceName, field, months = 12, options, fieldHa
|
|
|
6633
6978
|
const startDate = monthsAgo.toISOString().substring(0, 7);
|
|
6634
6979
|
const endDate = now.toISOString().substring(0, 7);
|
|
6635
6980
|
const data = await getAnalytics(resourceName, field, {
|
|
6981
|
+
...options,
|
|
6982
|
+
// ✅ Include all options (recordId, etc.)
|
|
6636
6983
|
period: "month",
|
|
6637
6984
|
startDate,
|
|
6638
6985
|
endDate
|
|
@@ -6654,13 +7001,15 @@ async function getLastNMonths(resourceName, field, months = 12, options, fieldHa
|
|
|
6654
7001
|
|
|
6655
7002
|
function addHelperMethods(resource, plugin, config) {
|
|
6656
7003
|
resource.set = async (id, field, value) => {
|
|
6657
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
7004
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
6658
7005
|
const now = /* @__PURE__ */ new Date();
|
|
6659
7006
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6660
7007
|
const transaction = {
|
|
6661
7008
|
id: idGenerator(),
|
|
6662
7009
|
originalId: id,
|
|
6663
7010
|
field: handler.field,
|
|
7011
|
+
fieldPath,
|
|
7012
|
+
// Store full path for nested access
|
|
6664
7013
|
value,
|
|
6665
7014
|
operation: "set",
|
|
6666
7015
|
timestamp: now.toISOString(),
|
|
@@ -6672,18 +7021,20 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6672
7021
|
};
|
|
6673
7022
|
await handler.transactionResource.insert(transaction);
|
|
6674
7023
|
if (config.mode === "sync") {
|
|
6675
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
7024
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6676
7025
|
}
|
|
6677
7026
|
return value;
|
|
6678
7027
|
};
|
|
6679
7028
|
resource.add = async (id, field, amount) => {
|
|
6680
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
7029
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6681
7030
|
const now = /* @__PURE__ */ new Date();
|
|
6682
7031
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6683
7032
|
const transaction = {
|
|
6684
7033
|
id: idGenerator(),
|
|
6685
7034
|
originalId: id,
|
|
6686
7035
|
field: handler.field,
|
|
7036
|
+
fieldPath,
|
|
7037
|
+
// Store full path for nested access
|
|
6687
7038
|
value: amount,
|
|
6688
7039
|
operation: "add",
|
|
6689
7040
|
timestamp: now.toISOString(),
|
|
@@ -6695,20 +7046,24 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6695
7046
|
};
|
|
6696
7047
|
await handler.transactionResource.insert(transaction);
|
|
6697
7048
|
if (config.mode === "sync") {
|
|
6698
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
7049
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6699
7050
|
}
|
|
6700
7051
|
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
6701
|
-
|
|
7052
|
+
if (!ok || !record) return amount;
|
|
7053
|
+
const lodash = await import('lodash-es');
|
|
7054
|
+
const currentValue = lodash.get(record, fieldPath, 0);
|
|
6702
7055
|
return currentValue + amount;
|
|
6703
7056
|
};
|
|
6704
7057
|
resource.sub = async (id, field, amount) => {
|
|
6705
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
7058
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6706
7059
|
const now = /* @__PURE__ */ new Date();
|
|
6707
7060
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6708
7061
|
const transaction = {
|
|
6709
7062
|
id: idGenerator(),
|
|
6710
7063
|
originalId: id,
|
|
6711
7064
|
field: handler.field,
|
|
7065
|
+
fieldPath,
|
|
7066
|
+
// Store full path for nested access
|
|
6712
7067
|
value: amount,
|
|
6713
7068
|
operation: "sub",
|
|
6714
7069
|
timestamp: now.toISOString(),
|
|
@@ -6720,10 +7075,12 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6720
7075
|
};
|
|
6721
7076
|
await handler.transactionResource.insert(transaction);
|
|
6722
7077
|
if (config.mode === "sync") {
|
|
6723
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
7078
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6724
7079
|
}
|
|
6725
7080
|
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
6726
|
-
|
|
7081
|
+
if (!ok || !record) return -amount;
|
|
7082
|
+
const lodash = await import('lodash-es');
|
|
7083
|
+
const currentValue = lodash.get(record, fieldPath, 0);
|
|
6727
7084
|
return currentValue - amount;
|
|
6728
7085
|
};
|
|
6729
7086
|
resource.consolidate = async (id, field) => {
|
|
@@ -6809,11 +7166,14 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6809
7166
|
id: "string|required",
|
|
6810
7167
|
originalId: "string|required",
|
|
6811
7168
|
field: "string|required",
|
|
7169
|
+
fieldPath: "string|optional",
|
|
7170
|
+
// Support for nested field paths (e.g., 'utmResults.medium')
|
|
6812
7171
|
value: "number|required",
|
|
6813
7172
|
operation: "string|required",
|
|
6814
7173
|
timestamp: "string|required",
|
|
6815
7174
|
cohortDate: "string|required",
|
|
6816
|
-
cohortHour: "string|
|
|
7175
|
+
cohortHour: "string|optional",
|
|
7176
|
+
// ✅ FIX BUG #2: Changed from required to optional for migration compatibility
|
|
6817
7177
|
cohortWeek: "string|optional",
|
|
6818
7178
|
cohortMonth: "string|optional",
|
|
6819
7179
|
source: "string|optional",
|
|
@@ -13391,7 +13751,7 @@ class Database extends EventEmitter {
|
|
|
13391
13751
|
this.id = idGenerator(7);
|
|
13392
13752
|
this.version = "1";
|
|
13393
13753
|
this.s3dbVersion = (() => {
|
|
13394
|
-
const [ok, err, version] = tryFn(() => true ? "11.0
|
|
13754
|
+
const [ok, err, version] = tryFn(() => true ? "11.2.0" : "latest");
|
|
13395
13755
|
return ok ? version : "latest";
|
|
13396
13756
|
})();
|
|
13397
13757
|
this.resources = {};
|