s3db.js 11.0.4 → 11.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -2
- package/dist/s3db.cjs.js +502 -84
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +502 -84
- package/dist/s3db.es.js.map +1 -1
- package/package.json +2 -1
- package/src/concerns/crypto.js +7 -14
- package/src/plugins/eventual-consistency/analytics.js +44 -22
- package/src/plugins/eventual-consistency/consolidation.js +228 -80
- package/src/plugins/eventual-consistency/helpers.js +24 -8
- package/src/plugins/eventual-consistency/index.js +73 -1
- package/src/plugins/eventual-consistency/install.js +1 -0
- package/src/plugins/eventual-consistency/utils.js +154 -4
- package/src/concerns/advanced-metadata-encoding.js +0 -440
package/dist/s3db.es.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
+
import crypto, { createHash } from 'crypto';
|
|
1
2
|
import { customAlphabet, urlAlphabet } from 'nanoid';
|
|
2
3
|
import EventEmitter from 'events';
|
|
3
4
|
import { mkdir, copyFile, unlink, stat, access, readdir, writeFile, readFile, rm } from 'fs/promises';
|
|
4
5
|
import fs, { createReadStream, createWriteStream } from 'fs';
|
|
5
6
|
import { pipeline } from 'stream/promises';
|
|
6
7
|
import path, { join } from 'path';
|
|
7
|
-
import crypto, { createHash } from 'crypto';
|
|
8
8
|
import zlib from 'node:zlib';
|
|
9
9
|
import os from 'os';
|
|
10
10
|
import jsonStableStringify from 'json-stable-stringify';
|
|
@@ -524,15 +524,7 @@ function tryFnSync(fn) {
|
|
|
524
524
|
async function dynamicCrypto() {
|
|
525
525
|
let lib;
|
|
526
526
|
if (typeof process !== "undefined") {
|
|
527
|
-
|
|
528
|
-
const { webcrypto } = await import('crypto');
|
|
529
|
-
return webcrypto;
|
|
530
|
-
});
|
|
531
|
-
if (ok) {
|
|
532
|
-
lib = result;
|
|
533
|
-
} else {
|
|
534
|
-
throw new CryptoError("Crypto API not available", { original: err, context: "dynamicCrypto" });
|
|
535
|
-
}
|
|
527
|
+
lib = crypto.webcrypto;
|
|
536
528
|
} else if (typeof window !== "undefined") {
|
|
537
529
|
lib = window.crypto;
|
|
538
530
|
}
|
|
@@ -586,8 +578,7 @@ async function md5(data) {
|
|
|
586
578
|
throw new CryptoError("MD5 hashing is only available in Node.js environment", { context: "md5" });
|
|
587
579
|
}
|
|
588
580
|
const [ok, err, result] = await tryFn(async () => {
|
|
589
|
-
|
|
590
|
-
return createHash("md5").update(data).digest("base64");
|
|
581
|
+
return crypto.createHash("md5").update(data).digest("base64");
|
|
591
582
|
});
|
|
592
583
|
if (!ok) {
|
|
593
584
|
throw new CryptoError("MD5 hashing failed", { original: err, data });
|
|
@@ -5272,10 +5263,121 @@ function createFieldHandler(resourceName, fieldName) {
|
|
|
5272
5263
|
deferredSetup: false
|
|
5273
5264
|
};
|
|
5274
5265
|
}
|
|
5266
|
+
function validateNestedPath(resource, fieldPath) {
|
|
5267
|
+
const parts = fieldPath.split(".");
|
|
5268
|
+
const rootField = parts[0];
|
|
5269
|
+
if (!resource.attributes || !resource.attributes[rootField]) {
|
|
5270
|
+
return {
|
|
5271
|
+
valid: false,
|
|
5272
|
+
rootField,
|
|
5273
|
+
fullPath: fieldPath,
|
|
5274
|
+
error: `Root field "${rootField}" not found in resource attributes`
|
|
5275
|
+
};
|
|
5276
|
+
}
|
|
5277
|
+
if (parts.length === 1) {
|
|
5278
|
+
return { valid: true, rootField, fullPath: fieldPath };
|
|
5279
|
+
}
|
|
5280
|
+
let current = resource.attributes[rootField];
|
|
5281
|
+
let foundJson = false;
|
|
5282
|
+
let levelsAfterJson = 0;
|
|
5283
|
+
for (let i = 1; i < parts.length; i++) {
|
|
5284
|
+
const part = parts[i];
|
|
5285
|
+
if (foundJson) {
|
|
5286
|
+
levelsAfterJson++;
|
|
5287
|
+
if (levelsAfterJson > 1) {
|
|
5288
|
+
return {
|
|
5289
|
+
valid: false,
|
|
5290
|
+
rootField,
|
|
5291
|
+
fullPath: fieldPath,
|
|
5292
|
+
error: `Path "${fieldPath}" exceeds 1 level after 'json' field. Maximum nesting after 'json' is 1 level.`
|
|
5293
|
+
};
|
|
5294
|
+
}
|
|
5295
|
+
continue;
|
|
5296
|
+
}
|
|
5297
|
+
if (typeof current === "string") {
|
|
5298
|
+
if (current === "json" || current.startsWith("json|")) {
|
|
5299
|
+
foundJson = true;
|
|
5300
|
+
levelsAfterJson++;
|
|
5301
|
+
if (levelsAfterJson > 1) {
|
|
5302
|
+
return {
|
|
5303
|
+
valid: false,
|
|
5304
|
+
rootField,
|
|
5305
|
+
fullPath: fieldPath,
|
|
5306
|
+
error: `Path "${fieldPath}" exceeds 1 level after 'json' field`
|
|
5307
|
+
};
|
|
5308
|
+
}
|
|
5309
|
+
continue;
|
|
5310
|
+
}
|
|
5311
|
+
return {
|
|
5312
|
+
valid: false,
|
|
5313
|
+
rootField,
|
|
5314
|
+
fullPath: fieldPath,
|
|
5315
|
+
error: `Field "${parts.slice(0, i).join(".")}" is type "${current}" and cannot be nested`
|
|
5316
|
+
};
|
|
5317
|
+
}
|
|
5318
|
+
if (typeof current === "object") {
|
|
5319
|
+
if (current.$$type) {
|
|
5320
|
+
const type = current.$$type;
|
|
5321
|
+
if (type === "json" || type.includes("json")) {
|
|
5322
|
+
foundJson = true;
|
|
5323
|
+
levelsAfterJson++;
|
|
5324
|
+
continue;
|
|
5325
|
+
}
|
|
5326
|
+
if (type !== "object" && !type.includes("object")) {
|
|
5327
|
+
return {
|
|
5328
|
+
valid: false,
|
|
5329
|
+
rootField,
|
|
5330
|
+
fullPath: fieldPath,
|
|
5331
|
+
error: `Field "${parts.slice(0, i).join(".")}" is type "${type}" and cannot be nested`
|
|
5332
|
+
};
|
|
5333
|
+
}
|
|
5334
|
+
}
|
|
5335
|
+
if (!current[part]) {
|
|
5336
|
+
return {
|
|
5337
|
+
valid: false,
|
|
5338
|
+
rootField,
|
|
5339
|
+
fullPath: fieldPath,
|
|
5340
|
+
error: `Field "${part}" not found in "${parts.slice(0, i).join(".")}"`
|
|
5341
|
+
};
|
|
5342
|
+
}
|
|
5343
|
+
current = current[part];
|
|
5344
|
+
} else {
|
|
5345
|
+
return {
|
|
5346
|
+
valid: false,
|
|
5347
|
+
rootField,
|
|
5348
|
+
fullPath: fieldPath,
|
|
5349
|
+
error: `Invalid structure at "${parts.slice(0, i).join(".")}"`
|
|
5350
|
+
};
|
|
5351
|
+
}
|
|
5352
|
+
}
|
|
5353
|
+
return { valid: true, rootField, fullPath: fieldPath };
|
|
5354
|
+
}
|
|
5275
5355
|
function resolveFieldAndPlugin(resource, field, value) {
|
|
5276
5356
|
if (!resource._eventualConsistencyPlugins) {
|
|
5277
5357
|
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
5278
5358
|
}
|
|
5359
|
+
if (field.includes(".")) {
|
|
5360
|
+
const validation = validateNestedPath(resource, field);
|
|
5361
|
+
if (!validation.valid) {
|
|
5362
|
+
throw new Error(validation.error);
|
|
5363
|
+
}
|
|
5364
|
+
const rootField = validation.rootField;
|
|
5365
|
+
const fieldPlugin2 = resource._eventualConsistencyPlugins[rootField];
|
|
5366
|
+
if (!fieldPlugin2) {
|
|
5367
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5368
|
+
throw new Error(
|
|
5369
|
+
`No eventual consistency plugin found for root field "${rootField}". Available fields: ${availableFields}`
|
|
5370
|
+
);
|
|
5371
|
+
}
|
|
5372
|
+
return {
|
|
5373
|
+
field: rootField,
|
|
5374
|
+
// Root field for plugin lookup
|
|
5375
|
+
fieldPath: field,
|
|
5376
|
+
// Full path for nested access
|
|
5377
|
+
value,
|
|
5378
|
+
plugin: fieldPlugin2
|
|
5379
|
+
};
|
|
5380
|
+
}
|
|
5279
5381
|
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
5280
5382
|
if (!fieldPlugin) {
|
|
5281
5383
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
@@ -5283,7 +5385,7 @@ function resolveFieldAndPlugin(resource, field, value) {
|
|
|
5283
5385
|
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5284
5386
|
);
|
|
5285
5387
|
}
|
|
5286
|
-
return { field, value, plugin: fieldPlugin };
|
|
5388
|
+
return { field, fieldPath: field, value, plugin: fieldPlugin };
|
|
5287
5389
|
}
|
|
5288
5390
|
function groupByCohort(transactions, cohortField) {
|
|
5289
5391
|
const groups = {};
|
|
@@ -5528,10 +5630,10 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5528
5630
|
})
|
|
5529
5631
|
);
|
|
5530
5632
|
if (!ok || !transactions || transactions.length === 0) {
|
|
5531
|
-
const [
|
|
5633
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
5532
5634
|
() => targetResource.get(originalId)
|
|
5533
5635
|
);
|
|
5534
|
-
const currentValue2 =
|
|
5636
|
+
const currentValue2 = recordOk2 && record2 ? record2[config.field] || 0 : 0;
|
|
5535
5637
|
if (config.verbose) {
|
|
5536
5638
|
console.log(
|
|
5537
5639
|
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions for ${originalId}, skipping`
|
|
@@ -5566,6 +5668,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5566
5668
|
);
|
|
5567
5669
|
}
|
|
5568
5670
|
currentValue = 0;
|
|
5671
|
+
appliedTransactions.length = 0;
|
|
5569
5672
|
} else {
|
|
5570
5673
|
appliedTransactions.sort(
|
|
5571
5674
|
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
@@ -5573,42 +5676,46 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5573
5676
|
const hasSetInApplied = appliedTransactions.some((t) => t.operation === "set");
|
|
5574
5677
|
if (!hasSetInApplied) {
|
|
5575
5678
|
const recordValue = recordExists[config.field] || 0;
|
|
5576
|
-
|
|
5577
|
-
|
|
5578
|
-
|
|
5579
|
-
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
|
|
5593
|
-
|
|
5594
|
-
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5679
|
+
if (typeof recordValue === "number") {
|
|
5680
|
+
let appliedDelta = 0;
|
|
5681
|
+
for (const t of appliedTransactions) {
|
|
5682
|
+
if (t.operation === "add") appliedDelta += t.value;
|
|
5683
|
+
else if (t.operation === "sub") appliedDelta -= t.value;
|
|
5684
|
+
}
|
|
5685
|
+
const baseValue = recordValue - appliedDelta;
|
|
5686
|
+
const hasExistingAnchor = appliedTransactions.some((t) => t.source === "anchor");
|
|
5687
|
+
if (baseValue !== 0 && typeof baseValue === "number" && !hasExistingAnchor) {
|
|
5688
|
+
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
5689
|
+
const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
|
|
5690
|
+
const anchorTransaction = {
|
|
5691
|
+
id: idGenerator(),
|
|
5692
|
+
originalId,
|
|
5693
|
+
field: config.field,
|
|
5694
|
+
fieldPath: config.field,
|
|
5695
|
+
// Add fieldPath for consistency
|
|
5696
|
+
value: baseValue,
|
|
5697
|
+
operation: "set",
|
|
5698
|
+
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(),
|
|
5699
|
+
// 1ms before first txn to ensure it's first
|
|
5700
|
+
cohortDate: cohortInfo.date,
|
|
5701
|
+
cohortHour: cohortInfo.hour,
|
|
5702
|
+
cohortMonth: cohortInfo.month,
|
|
5703
|
+
source: "anchor",
|
|
5704
|
+
applied: true
|
|
5705
|
+
};
|
|
5706
|
+
await transactionResource.insert(anchorTransaction);
|
|
5707
|
+
appliedTransactions.unshift(anchorTransaction);
|
|
5708
|
+
}
|
|
5602
5709
|
}
|
|
5603
5710
|
}
|
|
5604
5711
|
currentValue = config.reducer(appliedTransactions);
|
|
5605
5712
|
}
|
|
5606
5713
|
} else {
|
|
5607
|
-
const [
|
|
5714
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
5608
5715
|
() => targetResource.get(originalId)
|
|
5609
5716
|
);
|
|
5610
|
-
currentValue =
|
|
5611
|
-
if (currentValue !== 0) {
|
|
5717
|
+
currentValue = recordOk2 && record2 ? record2[config.field] || 0 : 0;
|
|
5718
|
+
if (currentValue !== 0 && typeof currentValue === "number") {
|
|
5612
5719
|
let anchorTimestamp;
|
|
5613
5720
|
if (transactions && transactions.length > 0) {
|
|
5614
5721
|
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
@@ -5621,6 +5728,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5621
5728
|
id: idGenerator(),
|
|
5622
5729
|
originalId,
|
|
5623
5730
|
field: config.field,
|
|
5731
|
+
fieldPath: config.field,
|
|
5732
|
+
// Add fieldPath for consistency
|
|
5624
5733
|
value: currentValue,
|
|
5625
5734
|
operation: "set",
|
|
5626
5735
|
timestamp: anchorTimestamp,
|
|
@@ -5646,38 +5755,101 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5646
5755
|
transactions.sort(
|
|
5647
5756
|
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5648
5757
|
);
|
|
5649
|
-
const
|
|
5650
|
-
|
|
5651
|
-
|
|
5758
|
+
const transactionsByPath = {};
|
|
5759
|
+
for (const txn of transactions) {
|
|
5760
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
5761
|
+
if (!transactionsByPath[path]) {
|
|
5762
|
+
transactionsByPath[path] = [];
|
|
5763
|
+
}
|
|
5764
|
+
transactionsByPath[path].push(txn);
|
|
5652
5765
|
}
|
|
5653
|
-
const
|
|
5654
|
-
if (
|
|
5655
|
-
|
|
5656
|
-
|
|
5657
|
-
|
|
5766
|
+
const appliedByPath = {};
|
|
5767
|
+
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
5768
|
+
for (const txn of appliedTransactions) {
|
|
5769
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
5770
|
+
if (!appliedByPath[path]) {
|
|
5771
|
+
appliedByPath[path] = [];
|
|
5772
|
+
}
|
|
5773
|
+
appliedByPath[path].push(txn);
|
|
5774
|
+
}
|
|
5775
|
+
}
|
|
5776
|
+
const consolidatedValues = {};
|
|
5777
|
+
const lodash = await import('lodash-es');
|
|
5778
|
+
const [currentRecordOk, currentRecordErr, currentRecord] = await tryFn(
|
|
5779
|
+
() => targetResource.get(originalId)
|
|
5780
|
+
);
|
|
5781
|
+
for (const [fieldPath, pathTransactions] of Object.entries(transactionsByPath)) {
|
|
5782
|
+
let pathCurrentValue = 0;
|
|
5783
|
+
if (appliedByPath[fieldPath] && appliedByPath[fieldPath].length > 0) {
|
|
5784
|
+
appliedByPath[fieldPath].sort(
|
|
5785
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5786
|
+
);
|
|
5787
|
+
pathCurrentValue = config.reducer(appliedByPath[fieldPath]);
|
|
5788
|
+
} else {
|
|
5789
|
+
if (currentRecordOk && currentRecord) {
|
|
5790
|
+
const recordValue = lodash.get(currentRecord, fieldPath, 0);
|
|
5791
|
+
if (typeof recordValue === "number") {
|
|
5792
|
+
pathCurrentValue = recordValue;
|
|
5793
|
+
}
|
|
5794
|
+
}
|
|
5795
|
+
}
|
|
5796
|
+
if (pathCurrentValue !== 0) {
|
|
5797
|
+
pathTransactions.unshift(createSyntheticSetTransaction(pathCurrentValue));
|
|
5798
|
+
}
|
|
5799
|
+
const pathConsolidatedValue = config.reducer(pathTransactions);
|
|
5800
|
+
consolidatedValues[fieldPath] = pathConsolidatedValue;
|
|
5801
|
+
if (config.verbose) {
|
|
5802
|
+
console.log(
|
|
5803
|
+
`[EventualConsistency] ${config.resource}.${fieldPath} - ${originalId}: ${pathCurrentValue} \u2192 ${pathConsolidatedValue} (${pathTransactions.length - (pathCurrentValue !== 0 ? 1 : 0)} pending txns)`
|
|
5804
|
+
);
|
|
5805
|
+
}
|
|
5658
5806
|
}
|
|
5659
5807
|
if (config.verbose) {
|
|
5660
5808
|
console.log(
|
|
5661
5809
|
`\u{1F525} [DEBUG] BEFORE targetResource.update() {
|
|
5662
5810
|
originalId: '${originalId}',
|
|
5663
|
-
|
|
5664
|
-
consolidatedValue: ${consolidatedValue},
|
|
5665
|
-
currentValue: ${currentValue}
|
|
5811
|
+
consolidatedValues: ${JSON.stringify(consolidatedValues, null, 2)}
|
|
5666
5812
|
}`
|
|
5667
5813
|
);
|
|
5668
5814
|
}
|
|
5669
|
-
const [
|
|
5670
|
-
() => targetResource.
|
|
5671
|
-
[config.field]: consolidatedValue
|
|
5672
|
-
})
|
|
5815
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
5816
|
+
() => targetResource.get(originalId)
|
|
5673
5817
|
);
|
|
5818
|
+
let updateOk, updateErr, updateResult;
|
|
5819
|
+
if (!recordOk || !record) {
|
|
5820
|
+
if (config.verbose) {
|
|
5821
|
+
console.log(
|
|
5822
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist yet. Will attempt update anyway (expected to fail).`
|
|
5823
|
+
);
|
|
5824
|
+
}
|
|
5825
|
+
const minimalRecord = { id: originalId };
|
|
5826
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
5827
|
+
lodash.set(minimalRecord, fieldPath, value);
|
|
5828
|
+
}
|
|
5829
|
+
const result = await tryFn(
|
|
5830
|
+
() => targetResource.update(originalId, minimalRecord)
|
|
5831
|
+
);
|
|
5832
|
+
updateOk = result[0];
|
|
5833
|
+
updateErr = result[1];
|
|
5834
|
+
updateResult = result[2];
|
|
5835
|
+
} else {
|
|
5836
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
5837
|
+
lodash.set(record, fieldPath, value);
|
|
5838
|
+
}
|
|
5839
|
+
const result = await tryFn(
|
|
5840
|
+
() => targetResource.update(originalId, record)
|
|
5841
|
+
);
|
|
5842
|
+
updateOk = result[0];
|
|
5843
|
+
updateErr = result[1];
|
|
5844
|
+
updateResult = result[2];
|
|
5845
|
+
}
|
|
5846
|
+
const consolidatedValue = consolidatedValues[config.field] || (record ? lodash.get(record, config.field, 0) : 0);
|
|
5674
5847
|
if (config.verbose) {
|
|
5675
5848
|
console.log(
|
|
5676
5849
|
`\u{1F525} [DEBUG] AFTER targetResource.update() {
|
|
5677
5850
|
updateOk: ${updateOk},
|
|
5678
5851
|
updateErr: ${updateErr?.message || "undefined"},
|
|
5679
|
-
|
|
5680
|
-
hasField: ${updateResult?.[config.field]}
|
|
5852
|
+
consolidatedValue (main field): ${consolidatedValue}
|
|
5681
5853
|
}`
|
|
5682
5854
|
);
|
|
5683
5855
|
}
|
|
@@ -5685,24 +5857,27 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5685
5857
|
const [verifyOk, verifyErr, verifiedRecord] = await tryFn(
|
|
5686
5858
|
() => targetResource.get(originalId, { skipCache: true })
|
|
5687
5859
|
);
|
|
5688
|
-
|
|
5689
|
-
|
|
5690
|
-
|
|
5691
|
-
|
|
5692
|
-
|
|
5693
|
-
|
|
5860
|
+
for (const [fieldPath, expectedValue] of Object.entries(consolidatedValues)) {
|
|
5861
|
+
const actualValue = lodash.get(verifiedRecord, fieldPath);
|
|
5862
|
+
const match = actualValue === expectedValue;
|
|
5863
|
+
console.log(
|
|
5864
|
+
`\u{1F525} [DEBUG] VERIFICATION ${fieldPath} {
|
|
5865
|
+
expectedValue: ${expectedValue},
|
|
5866
|
+
actualValue: ${actualValue},
|
|
5867
|
+
${match ? "\u2705 MATCH" : "\u274C MISMATCH"}
|
|
5694
5868
|
}`
|
|
5695
|
-
|
|
5696
|
-
|
|
5697
|
-
|
|
5698
|
-
|
|
5869
|
+
);
|
|
5870
|
+
if (!match) {
|
|
5871
|
+
console.error(
|
|
5872
|
+
`\u274C [CRITICAL BUG] Update reported success but value not persisted!
|
|
5699
5873
|
Resource: ${config.resource}
|
|
5700
|
-
|
|
5874
|
+
FieldPath: ${fieldPath}
|
|
5701
5875
|
Record ID: ${originalId}
|
|
5702
|
-
Expected: ${
|
|
5703
|
-
Actually got: ${
|
|
5876
|
+
Expected: ${expectedValue}
|
|
5877
|
+
Actually got: ${actualValue}
|
|
5704
5878
|
This indicates a bug in s3db.js resource.update()`
|
|
5705
|
-
|
|
5879
|
+
);
|
|
5880
|
+
}
|
|
5706
5881
|
}
|
|
5707
5882
|
}
|
|
5708
5883
|
if (!updateOk) {
|
|
@@ -5887,6 +6062,38 @@ async function recalculateRecord(originalId, transactionResource, targetResource
|
|
|
5887
6062
|
`[EventualConsistency] ${config.resource}.${config.field} - Found ${allTransactions.length} total transactions for ${originalId}, marking all as pending...`
|
|
5888
6063
|
);
|
|
5889
6064
|
}
|
|
6065
|
+
const hasAnchor = allTransactions.some((txn) => txn.source === "anchor");
|
|
6066
|
+
if (!hasAnchor) {
|
|
6067
|
+
const now = /* @__PURE__ */ new Date();
|
|
6068
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6069
|
+
const oldestTransaction = allTransactions.sort(
|
|
6070
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
6071
|
+
)[0];
|
|
6072
|
+
const anchorTimestamp = oldestTransaction ? new Date(new Date(oldestTransaction.timestamp).getTime() - 1).toISOString() : now.toISOString();
|
|
6073
|
+
const anchorCohortInfo = getCohortInfo(new Date(anchorTimestamp), config.cohort.timezone, config.verbose);
|
|
6074
|
+
const anchorTransaction = {
|
|
6075
|
+
id: idGenerator(),
|
|
6076
|
+
originalId,
|
|
6077
|
+
field: config.field,
|
|
6078
|
+
fieldPath: config.field,
|
|
6079
|
+
value: 0,
|
|
6080
|
+
// Always 0 for recalculate - we start from scratch
|
|
6081
|
+
operation: "set",
|
|
6082
|
+
timestamp: anchorTimestamp,
|
|
6083
|
+
cohortDate: anchorCohortInfo.date,
|
|
6084
|
+
cohortHour: anchorCohortInfo.hour,
|
|
6085
|
+
cohortMonth: anchorCohortInfo.month,
|
|
6086
|
+
source: "anchor",
|
|
6087
|
+
applied: true
|
|
6088
|
+
// Anchor is always applied
|
|
6089
|
+
};
|
|
6090
|
+
await transactionResource.insert(anchorTransaction);
|
|
6091
|
+
if (config.verbose) {
|
|
6092
|
+
console.log(
|
|
6093
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Created anchor transaction for ${originalId} with value 0`
|
|
6094
|
+
);
|
|
6095
|
+
}
|
|
6096
|
+
}
|
|
5890
6097
|
const transactionsToReset = allTransactions.filter((txn) => txn.source !== "anchor");
|
|
5891
6098
|
const { results, errors } = await PromisePool.for(transactionsToReset).withConcurrency(10).process(async (txn) => {
|
|
5892
6099
|
const [ok, err] = await tryFn(
|
|
@@ -6514,16 +6721,151 @@ async function getTopRecords(resourceName, field, options, fieldHandlers) {
|
|
|
6514
6721
|
});
|
|
6515
6722
|
return records.slice(0, limit);
|
|
6516
6723
|
}
|
|
6724
|
+
async function getYearByDay(resourceName, field, year, options, fieldHandlers) {
|
|
6725
|
+
const startDate = `${year}-01-01`;
|
|
6726
|
+
const endDate = `${year}-12-31`;
|
|
6727
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6728
|
+
period: "day",
|
|
6729
|
+
startDate,
|
|
6730
|
+
endDate
|
|
6731
|
+
}, fieldHandlers);
|
|
6732
|
+
if (options.fillGaps) {
|
|
6733
|
+
return fillGaps(data, "day", startDate, endDate);
|
|
6734
|
+
}
|
|
6735
|
+
return data;
|
|
6736
|
+
}
|
|
6737
|
+
async function getWeekByDay(resourceName, field, week, options, fieldHandlers) {
|
|
6738
|
+
const year = parseInt(week.substring(0, 4));
|
|
6739
|
+
const weekNum = parseInt(week.substring(6, 8));
|
|
6740
|
+
const jan4 = new Date(Date.UTC(year, 0, 4));
|
|
6741
|
+
const jan4Day = jan4.getUTCDay() || 7;
|
|
6742
|
+
const firstMonday = new Date(Date.UTC(year, 0, 4 - jan4Day + 1));
|
|
6743
|
+
const weekStart = new Date(firstMonday);
|
|
6744
|
+
weekStart.setUTCDate(weekStart.getUTCDate() + (weekNum - 1) * 7);
|
|
6745
|
+
const days = [];
|
|
6746
|
+
for (let i = 0; i < 7; i++) {
|
|
6747
|
+
const day = new Date(weekStart);
|
|
6748
|
+
day.setUTCDate(weekStart.getUTCDate() + i);
|
|
6749
|
+
days.push(day.toISOString().substring(0, 10));
|
|
6750
|
+
}
|
|
6751
|
+
const startDate = days[0];
|
|
6752
|
+
const endDate = days[6];
|
|
6753
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6754
|
+
period: "day",
|
|
6755
|
+
startDate,
|
|
6756
|
+
endDate
|
|
6757
|
+
}, fieldHandlers);
|
|
6758
|
+
if (options.fillGaps) {
|
|
6759
|
+
return fillGaps(data, "day", startDate, endDate);
|
|
6760
|
+
}
|
|
6761
|
+
return data;
|
|
6762
|
+
}
|
|
6763
|
+
async function getWeekByHour(resourceName, field, week, options, fieldHandlers) {
|
|
6764
|
+
const year = parseInt(week.substring(0, 4));
|
|
6765
|
+
const weekNum = parseInt(week.substring(6, 8));
|
|
6766
|
+
const jan4 = new Date(Date.UTC(year, 0, 4));
|
|
6767
|
+
const jan4Day = jan4.getUTCDay() || 7;
|
|
6768
|
+
const firstMonday = new Date(Date.UTC(year, 0, 4 - jan4Day + 1));
|
|
6769
|
+
const weekStart = new Date(firstMonday);
|
|
6770
|
+
weekStart.setUTCDate(weekStart.getUTCDate() + (weekNum - 1) * 7);
|
|
6771
|
+
const weekEnd = new Date(weekStart);
|
|
6772
|
+
weekEnd.setUTCDate(weekEnd.getUTCDate() + 6);
|
|
6773
|
+
const startDate = weekStart.toISOString().substring(0, 10);
|
|
6774
|
+
const endDate = weekEnd.toISOString().substring(0, 10);
|
|
6775
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6776
|
+
period: "hour",
|
|
6777
|
+
startDate,
|
|
6778
|
+
endDate
|
|
6779
|
+
}, fieldHandlers);
|
|
6780
|
+
if (options.fillGaps) {
|
|
6781
|
+
return fillGaps(data, "hour", startDate, endDate);
|
|
6782
|
+
}
|
|
6783
|
+
return data;
|
|
6784
|
+
}
|
|
6785
|
+
async function getLastNHours(resourceName, field, hours = 24, options, fieldHandlers) {
|
|
6786
|
+
const now = /* @__PURE__ */ new Date();
|
|
6787
|
+
const hoursAgo = new Date(now);
|
|
6788
|
+
hoursAgo.setHours(hoursAgo.getHours() - hours + 1);
|
|
6789
|
+
const startHour = hoursAgo.toISOString().substring(0, 13);
|
|
6790
|
+
const endHour = now.toISOString().substring(0, 13);
|
|
6791
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6792
|
+
period: "hour",
|
|
6793
|
+
startDate: startHour,
|
|
6794
|
+
endDate: endHour
|
|
6795
|
+
}, fieldHandlers);
|
|
6796
|
+
if (options.fillGaps) {
|
|
6797
|
+
const result = [];
|
|
6798
|
+
const emptyRecord = { count: 0, sum: 0, avg: 0, min: 0, max: 0, recordCount: 0 };
|
|
6799
|
+
const dataMap = new Map(data.map((d) => [d.cohort, d]));
|
|
6800
|
+
const current = new Date(hoursAgo);
|
|
6801
|
+
for (let i = 0; i < hours; i++) {
|
|
6802
|
+
const cohort = current.toISOString().substring(0, 13);
|
|
6803
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
6804
|
+
current.setHours(current.getHours() + 1);
|
|
6805
|
+
}
|
|
6806
|
+
return result;
|
|
6807
|
+
}
|
|
6808
|
+
return data;
|
|
6809
|
+
}
|
|
6810
|
+
async function getLastNWeeks(resourceName, field, weeks = 4, options, fieldHandlers) {
|
|
6811
|
+
const now = /* @__PURE__ */ new Date();
|
|
6812
|
+
const weeksAgo = new Date(now);
|
|
6813
|
+
weeksAgo.setDate(weeksAgo.getDate() - weeks * 7);
|
|
6814
|
+
const weekCohorts = [];
|
|
6815
|
+
const currentDate = new Date(weeksAgo);
|
|
6816
|
+
while (currentDate <= now) {
|
|
6817
|
+
const weekCohort = getCohortWeekFromDate(currentDate);
|
|
6818
|
+
if (!weekCohorts.includes(weekCohort)) {
|
|
6819
|
+
weekCohorts.push(weekCohort);
|
|
6820
|
+
}
|
|
6821
|
+
currentDate.setDate(currentDate.getDate() + 7);
|
|
6822
|
+
}
|
|
6823
|
+
const startWeek = weekCohorts[0];
|
|
6824
|
+
const endWeek = weekCohorts[weekCohorts.length - 1];
|
|
6825
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6826
|
+
period: "week",
|
|
6827
|
+
startDate: startWeek,
|
|
6828
|
+
endDate: endWeek
|
|
6829
|
+
}, fieldHandlers);
|
|
6830
|
+
return data;
|
|
6831
|
+
}
|
|
6832
|
+
async function getLastNMonths(resourceName, field, months = 12, options, fieldHandlers) {
|
|
6833
|
+
const now = /* @__PURE__ */ new Date();
|
|
6834
|
+
const monthsAgo = new Date(now);
|
|
6835
|
+
monthsAgo.setMonth(monthsAgo.getMonth() - months + 1);
|
|
6836
|
+
const startDate = monthsAgo.toISOString().substring(0, 7);
|
|
6837
|
+
const endDate = now.toISOString().substring(0, 7);
|
|
6838
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6839
|
+
period: "month",
|
|
6840
|
+
startDate,
|
|
6841
|
+
endDate
|
|
6842
|
+
}, fieldHandlers);
|
|
6843
|
+
if (options.fillGaps) {
|
|
6844
|
+
const result = [];
|
|
6845
|
+
const emptyRecord = { count: 0, sum: 0, avg: 0, min: 0, max: 0, recordCount: 0 };
|
|
6846
|
+
const dataMap = new Map(data.map((d) => [d.cohort, d]));
|
|
6847
|
+
const current = new Date(monthsAgo);
|
|
6848
|
+
for (let i = 0; i < months; i++) {
|
|
6849
|
+
const cohort = current.toISOString().substring(0, 7);
|
|
6850
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
6851
|
+
current.setMonth(current.getMonth() + 1);
|
|
6852
|
+
}
|
|
6853
|
+
return result;
|
|
6854
|
+
}
|
|
6855
|
+
return data;
|
|
6856
|
+
}
|
|
6517
6857
|
|
|
6518
6858
|
function addHelperMethods(resource, plugin, config) {
|
|
6519
6859
|
resource.set = async (id, field, value) => {
|
|
6520
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
6860
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
6521
6861
|
const now = /* @__PURE__ */ new Date();
|
|
6522
6862
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6523
6863
|
const transaction = {
|
|
6524
6864
|
id: idGenerator(),
|
|
6525
6865
|
originalId: id,
|
|
6526
6866
|
field: handler.field,
|
|
6867
|
+
fieldPath,
|
|
6868
|
+
// Store full path for nested access
|
|
6527
6869
|
value,
|
|
6528
6870
|
operation: "set",
|
|
6529
6871
|
timestamp: now.toISOString(),
|
|
@@ -6535,18 +6877,20 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6535
6877
|
};
|
|
6536
6878
|
await handler.transactionResource.insert(transaction);
|
|
6537
6879
|
if (config.mode === "sync") {
|
|
6538
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
6880
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6539
6881
|
}
|
|
6540
6882
|
return value;
|
|
6541
6883
|
};
|
|
6542
6884
|
resource.add = async (id, field, amount) => {
|
|
6543
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6885
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6544
6886
|
const now = /* @__PURE__ */ new Date();
|
|
6545
6887
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6546
6888
|
const transaction = {
|
|
6547
6889
|
id: idGenerator(),
|
|
6548
6890
|
originalId: id,
|
|
6549
6891
|
field: handler.field,
|
|
6892
|
+
fieldPath,
|
|
6893
|
+
// Store full path for nested access
|
|
6550
6894
|
value: amount,
|
|
6551
6895
|
operation: "add",
|
|
6552
6896
|
timestamp: now.toISOString(),
|
|
@@ -6558,20 +6902,24 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6558
6902
|
};
|
|
6559
6903
|
await handler.transactionResource.insert(transaction);
|
|
6560
6904
|
if (config.mode === "sync") {
|
|
6561
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
6905
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6562
6906
|
}
|
|
6563
6907
|
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
6564
|
-
|
|
6908
|
+
if (!ok || !record) return amount;
|
|
6909
|
+
const lodash = await import('lodash-es');
|
|
6910
|
+
const currentValue = lodash.get(record, fieldPath, 0);
|
|
6565
6911
|
return currentValue + amount;
|
|
6566
6912
|
};
|
|
6567
6913
|
resource.sub = async (id, field, amount) => {
|
|
6568
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6914
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6569
6915
|
const now = /* @__PURE__ */ new Date();
|
|
6570
6916
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6571
6917
|
const transaction = {
|
|
6572
6918
|
id: idGenerator(),
|
|
6573
6919
|
originalId: id,
|
|
6574
6920
|
field: handler.field,
|
|
6921
|
+
fieldPath,
|
|
6922
|
+
// Store full path for nested access
|
|
6575
6923
|
value: amount,
|
|
6576
6924
|
operation: "sub",
|
|
6577
6925
|
timestamp: now.toISOString(),
|
|
@@ -6583,10 +6931,12 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6583
6931
|
};
|
|
6584
6932
|
await handler.transactionResource.insert(transaction);
|
|
6585
6933
|
if (config.mode === "sync") {
|
|
6586
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
6934
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6587
6935
|
}
|
|
6588
6936
|
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
6589
|
-
|
|
6937
|
+
if (!ok || !record) return -amount;
|
|
6938
|
+
const lodash = await import('lodash-es');
|
|
6939
|
+
const currentValue = lodash.get(record, fieldPath, 0);
|
|
6590
6940
|
return currentValue - amount;
|
|
6591
6941
|
};
|
|
6592
6942
|
resource.consolidate = async (id, field) => {
|
|
@@ -6672,6 +7022,8 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6672
7022
|
id: "string|required",
|
|
6673
7023
|
originalId: "string|required",
|
|
6674
7024
|
field: "string|required",
|
|
7025
|
+
fieldPath: "string|optional",
|
|
7026
|
+
// Support for nested field paths (e.g., 'utmResults.medium')
|
|
6675
7027
|
value: "number|required",
|
|
6676
7028
|
operation: "string|required",
|
|
6677
7029
|
timestamp: "string|required",
|
|
@@ -7170,6 +7522,72 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
7170
7522
|
async getTopRecords(resourceName, field, options = {}) {
|
|
7171
7523
|
return await getTopRecords(resourceName, field, options, this.fieldHandlers);
|
|
7172
7524
|
}
|
|
7525
|
+
/**
|
|
7526
|
+
* Get analytics for entire year, broken down by days
|
|
7527
|
+
* @param {string} resourceName - Resource name
|
|
7528
|
+
* @param {string} field - Field name
|
|
7529
|
+
* @param {number} year - Year (e.g., 2025)
|
|
7530
|
+
* @param {Object} options - Options
|
|
7531
|
+
* @returns {Promise<Array>} Daily analytics for the year (up to 365/366 records)
|
|
7532
|
+
*/
|
|
7533
|
+
async getYearByDay(resourceName, field, year, options = {}) {
|
|
7534
|
+
return await getYearByDay(resourceName, field, year, options, this.fieldHandlers);
|
|
7535
|
+
}
|
|
7536
|
+
/**
|
|
7537
|
+
* Get analytics for entire week, broken down by days
|
|
7538
|
+
* @param {string} resourceName - Resource name
|
|
7539
|
+
* @param {string} field - Field name
|
|
7540
|
+
* @param {string} week - Week in YYYY-Www format (e.g., '2025-W42')
|
|
7541
|
+
* @param {Object} options - Options
|
|
7542
|
+
* @returns {Promise<Array>} Daily analytics for the week (7 records)
|
|
7543
|
+
*/
|
|
7544
|
+
async getWeekByDay(resourceName, field, week, options = {}) {
|
|
7545
|
+
return await getWeekByDay(resourceName, field, week, options, this.fieldHandlers);
|
|
7546
|
+
}
|
|
7547
|
+
/**
|
|
7548
|
+
* Get analytics for entire week, broken down by hours
|
|
7549
|
+
* @param {string} resourceName - Resource name
|
|
7550
|
+
* @param {string} field - Field name
|
|
7551
|
+
* @param {string} week - Week in YYYY-Www format (e.g., '2025-W42')
|
|
7552
|
+
* @param {Object} options - Options
|
|
7553
|
+
* @returns {Promise<Array>} Hourly analytics for the week (168 records)
|
|
7554
|
+
*/
|
|
7555
|
+
async getWeekByHour(resourceName, field, week, options = {}) {
|
|
7556
|
+
return await getWeekByHour(resourceName, field, week, options, this.fieldHandlers);
|
|
7557
|
+
}
|
|
7558
|
+
/**
|
|
7559
|
+
* Get analytics for last N hours
|
|
7560
|
+
* @param {string} resourceName - Resource name
|
|
7561
|
+
* @param {string} field - Field name
|
|
7562
|
+
* @param {number} hours - Number of hours to look back (default: 24)
|
|
7563
|
+
* @param {Object} options - Options
|
|
7564
|
+
* @returns {Promise<Array>} Hourly analytics
|
|
7565
|
+
*/
|
|
7566
|
+
async getLastNHours(resourceName, field, hours = 24, options = {}) {
|
|
7567
|
+
return await getLastNHours(resourceName, field, hours, options, this.fieldHandlers);
|
|
7568
|
+
}
|
|
7569
|
+
/**
|
|
7570
|
+
* Get analytics for last N weeks
|
|
7571
|
+
* @param {string} resourceName - Resource name
|
|
7572
|
+
* @param {string} field - Field name
|
|
7573
|
+
* @param {number} weeks - Number of weeks to look back (default: 4)
|
|
7574
|
+
* @param {Object} options - Options
|
|
7575
|
+
* @returns {Promise<Array>} Weekly analytics
|
|
7576
|
+
*/
|
|
7577
|
+
async getLastNWeeks(resourceName, field, weeks = 4, options = {}) {
|
|
7578
|
+
return await getLastNWeeks(resourceName, field, weeks, options, this.fieldHandlers);
|
|
7579
|
+
}
|
|
7580
|
+
/**
|
|
7581
|
+
* Get analytics for last N months
|
|
7582
|
+
* @param {string} resourceName - Resource name
|
|
7583
|
+
* @param {string} field - Field name
|
|
7584
|
+
* @param {number} months - Number of months to look back (default: 12)
|
|
7585
|
+
* @param {Object} options - Options
|
|
7586
|
+
* @returns {Promise<Array>} Monthly analytics
|
|
7587
|
+
*/
|
|
7588
|
+
async getLastNMonths(resourceName, field, months = 12, options = {}) {
|
|
7589
|
+
return await getLastNMonths(resourceName, field, months, options, this.fieldHandlers);
|
|
7590
|
+
}
|
|
7173
7591
|
}
|
|
7174
7592
|
|
|
7175
7593
|
class FullTextPlugin extends Plugin {
|
|
@@ -13188,7 +13606,7 @@ class Database extends EventEmitter {
|
|
|
13188
13606
|
this.id = idGenerator(7);
|
|
13189
13607
|
this.version = "1";
|
|
13190
13608
|
this.s3dbVersion = (() => {
|
|
13191
|
-
const [ok, err, version] = tryFn(() => true ? "11.0
|
|
13609
|
+
const [ok, err, version] = tryFn(() => true ? "11.1.0" : "latest");
|
|
13192
13610
|
return ok ? version : "latest";
|
|
13193
13611
|
})();
|
|
13194
13612
|
this.resources = {};
|