s3db.js 11.0.5 → 11.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -2
- package/dist/s3db.cjs.js +303 -84
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +303 -84
- package/dist/s3db.es.js.map +1 -1
- package/package.json +2 -1
- package/src/concerns/crypto.js +7 -14
- package/src/plugins/eventual-consistency/consolidation.js +228 -80
- package/src/plugins/eventual-consistency/helpers.js +24 -8
- package/src/plugins/eventual-consistency/install.js +1 -0
- package/src/plugins/eventual-consistency/utils.js +154 -4
- package/src/concerns/advanced-metadata-encoding.js +0 -440
package/README.md
CHANGED
|
@@ -794,7 +794,7 @@ await users.insert({ name: "John", email: "john@example.com" });
|
|
|
794
794
|
- **📝 [Audit Plugin](./docs/plugins/audit.md)** - Comprehensive audit logging for compliance
|
|
795
795
|
- **📬 [Queue Consumer Plugin](./docs/plugins/queue-consumer.md)** - Message consumption from SQS/RabbitMQ
|
|
796
796
|
- **🔒 [S3Queue Plugin](./docs/plugins/s3-queue.md)** - Distributed queue processing with zero race conditions
|
|
797
|
-
- **📈 [Eventual Consistency Plugin](./docs/plugins/eventual-consistency.md)** -
|
|
797
|
+
- **📈 [Eventual Consistency Plugin](./docs/plugins/eventual-consistency.md)** - Transactional counters with pre-computed analytics (15 functions for time-series data)
|
|
798
798
|
- **📅 [Scheduler Plugin](./docs/plugins/scheduler.md)** - Task scheduling and automation
|
|
799
799
|
- **🔄 [State Machine Plugin](./docs/plugins/state-machine.md)** - State management and transitions
|
|
800
800
|
- **💾 [Backup Plugin](./docs/plugins/backup.md)** - Backup and restore functionality
|
|
@@ -1868,6 +1868,12 @@ await users.insert({ name: 'John' });
|
|
|
1868
1868
|
|
|
1869
1869
|
## 📖 API Reference
|
|
1870
1870
|
|
|
1871
|
+
### 📚 Core Classes Documentation
|
|
1872
|
+
|
|
1873
|
+
- **[Client Class](./docs/client.md)** - Low-level S3 operations, HTTP client configuration, and advanced object management
|
|
1874
|
+
- **[Database Class](./docs/database.md)** - High-level database interface (coming soon)
|
|
1875
|
+
- **[Resource Class](./docs/resource.md)** - Resource operations and methods (coming soon)
|
|
1876
|
+
|
|
1871
1877
|
### 🔌 Database Operations
|
|
1872
1878
|
|
|
1873
1879
|
| Method | Description | Example |
|
|
@@ -1971,4 +1977,55 @@ console.log(`Total users: ${allUsers.length}`);
|
|
|
1971
1977
|
| Method | Description | Example |
|
|
1972
1978
|
|--------|-------------|---------|
|
|
1973
1979
|
| `readable(options?)` | Create readable stream | `await users.readable()` |
|
|
1974
|
-
| `writable(options?)` | Create writable stream | `await users.writable()` |
|
|
1980
|
+
| `writable(options?)` | Create writable stream | `await users.writable()` |
|
|
1981
|
+
|
|
1982
|
+
---
|
|
1983
|
+
|
|
1984
|
+
## 📊 Performance Benchmarks
|
|
1985
|
+
|
|
1986
|
+
> **⚠️ Important**: All benchmark results documented below were generated using **Node.js v22.6.0**. Performance results may vary with different Node.js versions.
|
|
1987
|
+
|
|
1988
|
+
s3db.js includes comprehensive benchmarks demonstrating real-world performance optimizations. Key areas tested:
|
|
1989
|
+
|
|
1990
|
+
### 🎯 Data Encoding & Compression
|
|
1991
|
+
|
|
1992
|
+
**[Base62 Encoding](./docs/benchmarks/base62.md)** - Number compression for S3 metadata
|
|
1993
|
+
- **40-46% space savings** for large numbers
|
|
1994
|
+
- **5x faster encoding** vs Base36
|
|
1995
|
+
- **Real-world impact**: More data fits in 2KB S3 metadata limit
|
|
1996
|
+
|
|
1997
|
+
**[Advanced Encoding](./docs/benchmarks/advanced-encoding.md)** - Multi-technique compression
|
|
1998
|
+
- **67% savings** on ISO timestamps (Unix Base62)
|
|
1999
|
+
- **33% savings** on UUIDs (Binary Base64)
|
|
2000
|
+
- **95% savings** on common values (Dictionary encoding)
|
|
2001
|
+
- **Overall**: 40-50% metadata reduction on typical datasets
|
|
2002
|
+
|
|
2003
|
+
**[Smart Encoding](./docs/benchmarks/smart-encoding.md)** - Intelligent encoding selection
|
|
2004
|
+
- **Automatic type detection** and optimal encoding selection
|
|
2005
|
+
- **2-3x faster** UTF-8 byte calculations with caching
|
|
2006
|
+
- **Lazy evaluation** for performance-critical paths
|
|
2007
|
+
|
|
2008
|
+
### 🔌 Plugin Performance
|
|
2009
|
+
|
|
2010
|
+
**[EventualConsistency Plugin](./docs/benchmarks/eventual-consistency.md)** - Transaction processing & analytics
|
|
2011
|
+
- **70-100% faster writes** with async partitions
|
|
2012
|
+
- **Parallel analytics updates** for high-throughput scenarios
|
|
2013
|
+
- **O(1) partition queries** vs O(n) full scans
|
|
2014
|
+
|
|
2015
|
+
### 🗂️ Partitioning Performance
|
|
2016
|
+
|
|
2017
|
+
**[Partitions Matrix Benchmark](./docs/benchmarks/partitions.md)** - Performance testing across partition configurations
|
|
2018
|
+
- **Test matrix**: 0-10 partitions × 1-10 attributes (110 combinations)
|
|
2019
|
+
- **Measurements**: Create, insert, query (partition & full scan)
|
|
2020
|
+
- **Insights**: Find optimal partition configuration for your use case
|
|
2021
|
+
- Run with: `pnpm run benchmark:partitions`
|
|
2022
|
+
|
|
2023
|
+
### 📖 Benchmark Documentation
|
|
2024
|
+
|
|
2025
|
+
All benchmarks include:
|
|
2026
|
+
- ✅ **TL;DR summary** - Quick results and recommendations
|
|
2027
|
+
- ✅ **Code examples** - Runnable benchmark scripts
|
|
2028
|
+
- ✅ **Performance metrics** - Real numbers with explanations
|
|
2029
|
+
- ✅ **Use cases** - When to apply each optimization
|
|
2030
|
+
|
|
2031
|
+
**[📋 Complete Benchmark Index](./docs/benchmarks/README.md)**
|
package/dist/s3db.cjs.js
CHANGED
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
4
|
|
|
5
|
+
var crypto = require('crypto');
|
|
5
6
|
var nanoid = require('nanoid');
|
|
6
7
|
var EventEmitter = require('events');
|
|
7
8
|
var promises = require('fs/promises');
|
|
8
9
|
var fs = require('fs');
|
|
9
10
|
var promises$1 = require('stream/promises');
|
|
10
11
|
var path = require('path');
|
|
11
|
-
var crypto = require('crypto');
|
|
12
12
|
var zlib = require('node:zlib');
|
|
13
13
|
var os = require('os');
|
|
14
14
|
var jsonStableStringify = require('json-stable-stringify');
|
|
@@ -528,15 +528,7 @@ function tryFnSync(fn) {
|
|
|
528
528
|
async function dynamicCrypto() {
|
|
529
529
|
let lib;
|
|
530
530
|
if (typeof process !== "undefined") {
|
|
531
|
-
|
|
532
|
-
const { webcrypto } = await import('crypto');
|
|
533
|
-
return webcrypto;
|
|
534
|
-
});
|
|
535
|
-
if (ok) {
|
|
536
|
-
lib = result;
|
|
537
|
-
} else {
|
|
538
|
-
throw new CryptoError("Crypto API not available", { original: err, context: "dynamicCrypto" });
|
|
539
|
-
}
|
|
531
|
+
lib = crypto.webcrypto;
|
|
540
532
|
} else if (typeof window !== "undefined") {
|
|
541
533
|
lib = window.crypto;
|
|
542
534
|
}
|
|
@@ -590,8 +582,7 @@ async function md5(data) {
|
|
|
590
582
|
throw new CryptoError("MD5 hashing is only available in Node.js environment", { context: "md5" });
|
|
591
583
|
}
|
|
592
584
|
const [ok, err, result] = await tryFn(async () => {
|
|
593
|
-
|
|
594
|
-
return createHash("md5").update(data).digest("base64");
|
|
585
|
+
return crypto.createHash("md5").update(data).digest("base64");
|
|
595
586
|
});
|
|
596
587
|
if (!ok) {
|
|
597
588
|
throw new CryptoError("MD5 hashing failed", { original: err, data });
|
|
@@ -5276,10 +5267,121 @@ function createFieldHandler(resourceName, fieldName) {
|
|
|
5276
5267
|
deferredSetup: false
|
|
5277
5268
|
};
|
|
5278
5269
|
}
|
|
5270
|
+
function validateNestedPath(resource, fieldPath) {
|
|
5271
|
+
const parts = fieldPath.split(".");
|
|
5272
|
+
const rootField = parts[0];
|
|
5273
|
+
if (!resource.attributes || !resource.attributes[rootField]) {
|
|
5274
|
+
return {
|
|
5275
|
+
valid: false,
|
|
5276
|
+
rootField,
|
|
5277
|
+
fullPath: fieldPath,
|
|
5278
|
+
error: `Root field "${rootField}" not found in resource attributes`
|
|
5279
|
+
};
|
|
5280
|
+
}
|
|
5281
|
+
if (parts.length === 1) {
|
|
5282
|
+
return { valid: true, rootField, fullPath: fieldPath };
|
|
5283
|
+
}
|
|
5284
|
+
let current = resource.attributes[rootField];
|
|
5285
|
+
let foundJson = false;
|
|
5286
|
+
let levelsAfterJson = 0;
|
|
5287
|
+
for (let i = 1; i < parts.length; i++) {
|
|
5288
|
+
const part = parts[i];
|
|
5289
|
+
if (foundJson) {
|
|
5290
|
+
levelsAfterJson++;
|
|
5291
|
+
if (levelsAfterJson > 1) {
|
|
5292
|
+
return {
|
|
5293
|
+
valid: false,
|
|
5294
|
+
rootField,
|
|
5295
|
+
fullPath: fieldPath,
|
|
5296
|
+
error: `Path "${fieldPath}" exceeds 1 level after 'json' field. Maximum nesting after 'json' is 1 level.`
|
|
5297
|
+
};
|
|
5298
|
+
}
|
|
5299
|
+
continue;
|
|
5300
|
+
}
|
|
5301
|
+
if (typeof current === "string") {
|
|
5302
|
+
if (current === "json" || current.startsWith("json|")) {
|
|
5303
|
+
foundJson = true;
|
|
5304
|
+
levelsAfterJson++;
|
|
5305
|
+
if (levelsAfterJson > 1) {
|
|
5306
|
+
return {
|
|
5307
|
+
valid: false,
|
|
5308
|
+
rootField,
|
|
5309
|
+
fullPath: fieldPath,
|
|
5310
|
+
error: `Path "${fieldPath}" exceeds 1 level after 'json' field`
|
|
5311
|
+
};
|
|
5312
|
+
}
|
|
5313
|
+
continue;
|
|
5314
|
+
}
|
|
5315
|
+
return {
|
|
5316
|
+
valid: false,
|
|
5317
|
+
rootField,
|
|
5318
|
+
fullPath: fieldPath,
|
|
5319
|
+
error: `Field "${parts.slice(0, i).join(".")}" is type "${current}" and cannot be nested`
|
|
5320
|
+
};
|
|
5321
|
+
}
|
|
5322
|
+
if (typeof current === "object") {
|
|
5323
|
+
if (current.$$type) {
|
|
5324
|
+
const type = current.$$type;
|
|
5325
|
+
if (type === "json" || type.includes("json")) {
|
|
5326
|
+
foundJson = true;
|
|
5327
|
+
levelsAfterJson++;
|
|
5328
|
+
continue;
|
|
5329
|
+
}
|
|
5330
|
+
if (type !== "object" && !type.includes("object")) {
|
|
5331
|
+
return {
|
|
5332
|
+
valid: false,
|
|
5333
|
+
rootField,
|
|
5334
|
+
fullPath: fieldPath,
|
|
5335
|
+
error: `Field "${parts.slice(0, i).join(".")}" is type "${type}" and cannot be nested`
|
|
5336
|
+
};
|
|
5337
|
+
}
|
|
5338
|
+
}
|
|
5339
|
+
if (!current[part]) {
|
|
5340
|
+
return {
|
|
5341
|
+
valid: false,
|
|
5342
|
+
rootField,
|
|
5343
|
+
fullPath: fieldPath,
|
|
5344
|
+
error: `Field "${part}" not found in "${parts.slice(0, i).join(".")}"`
|
|
5345
|
+
};
|
|
5346
|
+
}
|
|
5347
|
+
current = current[part];
|
|
5348
|
+
} else {
|
|
5349
|
+
return {
|
|
5350
|
+
valid: false,
|
|
5351
|
+
rootField,
|
|
5352
|
+
fullPath: fieldPath,
|
|
5353
|
+
error: `Invalid structure at "${parts.slice(0, i).join(".")}"`
|
|
5354
|
+
};
|
|
5355
|
+
}
|
|
5356
|
+
}
|
|
5357
|
+
return { valid: true, rootField, fullPath: fieldPath };
|
|
5358
|
+
}
|
|
5279
5359
|
function resolveFieldAndPlugin(resource, field, value) {
|
|
5280
5360
|
if (!resource._eventualConsistencyPlugins) {
|
|
5281
5361
|
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
5282
5362
|
}
|
|
5363
|
+
if (field.includes(".")) {
|
|
5364
|
+
const validation = validateNestedPath(resource, field);
|
|
5365
|
+
if (!validation.valid) {
|
|
5366
|
+
throw new Error(validation.error);
|
|
5367
|
+
}
|
|
5368
|
+
const rootField = validation.rootField;
|
|
5369
|
+
const fieldPlugin2 = resource._eventualConsistencyPlugins[rootField];
|
|
5370
|
+
if (!fieldPlugin2) {
|
|
5371
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5372
|
+
throw new Error(
|
|
5373
|
+
`No eventual consistency plugin found for root field "${rootField}". Available fields: ${availableFields}`
|
|
5374
|
+
);
|
|
5375
|
+
}
|
|
5376
|
+
return {
|
|
5377
|
+
field: rootField,
|
|
5378
|
+
// Root field for plugin lookup
|
|
5379
|
+
fieldPath: field,
|
|
5380
|
+
// Full path for nested access
|
|
5381
|
+
value,
|
|
5382
|
+
plugin: fieldPlugin2
|
|
5383
|
+
};
|
|
5384
|
+
}
|
|
5283
5385
|
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
5284
5386
|
if (!fieldPlugin) {
|
|
5285
5387
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
@@ -5287,7 +5389,7 @@ function resolveFieldAndPlugin(resource, field, value) {
|
|
|
5287
5389
|
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5288
5390
|
);
|
|
5289
5391
|
}
|
|
5290
|
-
return { field, value, plugin: fieldPlugin };
|
|
5392
|
+
return { field, fieldPath: field, value, plugin: fieldPlugin };
|
|
5291
5393
|
}
|
|
5292
5394
|
function groupByCohort(transactions, cohortField) {
|
|
5293
5395
|
const groups = {};
|
|
@@ -5532,10 +5634,10 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5532
5634
|
})
|
|
5533
5635
|
);
|
|
5534
5636
|
if (!ok || !transactions || transactions.length === 0) {
|
|
5535
|
-
const [
|
|
5637
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
5536
5638
|
() => targetResource.get(originalId)
|
|
5537
5639
|
);
|
|
5538
|
-
const currentValue2 =
|
|
5640
|
+
const currentValue2 = recordOk2 && record2 ? record2[config.field] || 0 : 0;
|
|
5539
5641
|
if (config.verbose) {
|
|
5540
5642
|
console.log(
|
|
5541
5643
|
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions for ${originalId}, skipping`
|
|
@@ -5570,6 +5672,7 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5570
5672
|
);
|
|
5571
5673
|
}
|
|
5572
5674
|
currentValue = 0;
|
|
5675
|
+
appliedTransactions.length = 0;
|
|
5573
5676
|
} else {
|
|
5574
5677
|
appliedTransactions.sort(
|
|
5575
5678
|
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
@@ -5577,42 +5680,46 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5577
5680
|
const hasSetInApplied = appliedTransactions.some((t) => t.operation === "set");
|
|
5578
5681
|
if (!hasSetInApplied) {
|
|
5579
5682
|
const recordValue = recordExists[config.field] || 0;
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
|
|
5593
|
-
|
|
5594
|
-
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5683
|
+
if (typeof recordValue === "number") {
|
|
5684
|
+
let appliedDelta = 0;
|
|
5685
|
+
for (const t of appliedTransactions) {
|
|
5686
|
+
if (t.operation === "add") appliedDelta += t.value;
|
|
5687
|
+
else if (t.operation === "sub") appliedDelta -= t.value;
|
|
5688
|
+
}
|
|
5689
|
+
const baseValue = recordValue - appliedDelta;
|
|
5690
|
+
const hasExistingAnchor = appliedTransactions.some((t) => t.source === "anchor");
|
|
5691
|
+
if (baseValue !== 0 && typeof baseValue === "number" && !hasExistingAnchor) {
|
|
5692
|
+
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
5693
|
+
const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
|
|
5694
|
+
const anchorTransaction = {
|
|
5695
|
+
id: idGenerator(),
|
|
5696
|
+
originalId,
|
|
5697
|
+
field: config.field,
|
|
5698
|
+
fieldPath: config.field,
|
|
5699
|
+
// Add fieldPath for consistency
|
|
5700
|
+
value: baseValue,
|
|
5701
|
+
operation: "set",
|
|
5702
|
+
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(),
|
|
5703
|
+
// 1ms before first txn to ensure it's first
|
|
5704
|
+
cohortDate: cohortInfo.date,
|
|
5705
|
+
cohortHour: cohortInfo.hour,
|
|
5706
|
+
cohortMonth: cohortInfo.month,
|
|
5707
|
+
source: "anchor",
|
|
5708
|
+
applied: true
|
|
5709
|
+
};
|
|
5710
|
+
await transactionResource.insert(anchorTransaction);
|
|
5711
|
+
appliedTransactions.unshift(anchorTransaction);
|
|
5712
|
+
}
|
|
5606
5713
|
}
|
|
5607
5714
|
}
|
|
5608
5715
|
currentValue = config.reducer(appliedTransactions);
|
|
5609
5716
|
}
|
|
5610
5717
|
} else {
|
|
5611
|
-
const [
|
|
5718
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
5612
5719
|
() => targetResource.get(originalId)
|
|
5613
5720
|
);
|
|
5614
|
-
currentValue =
|
|
5615
|
-
if (currentValue !== 0) {
|
|
5721
|
+
currentValue = recordOk2 && record2 ? record2[config.field] || 0 : 0;
|
|
5722
|
+
if (currentValue !== 0 && typeof currentValue === "number") {
|
|
5616
5723
|
let anchorTimestamp;
|
|
5617
5724
|
if (transactions && transactions.length > 0) {
|
|
5618
5725
|
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
@@ -5625,6 +5732,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5625
5732
|
id: idGenerator(),
|
|
5626
5733
|
originalId,
|
|
5627
5734
|
field: config.field,
|
|
5735
|
+
fieldPath: config.field,
|
|
5736
|
+
// Add fieldPath for consistency
|
|
5628
5737
|
value: currentValue,
|
|
5629
5738
|
operation: "set",
|
|
5630
5739
|
timestamp: anchorTimestamp,
|
|
@@ -5650,38 +5759,101 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5650
5759
|
transactions.sort(
|
|
5651
5760
|
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5652
5761
|
);
|
|
5653
|
-
const
|
|
5654
|
-
|
|
5655
|
-
|
|
5762
|
+
const transactionsByPath = {};
|
|
5763
|
+
for (const txn of transactions) {
|
|
5764
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
5765
|
+
if (!transactionsByPath[path]) {
|
|
5766
|
+
transactionsByPath[path] = [];
|
|
5767
|
+
}
|
|
5768
|
+
transactionsByPath[path].push(txn);
|
|
5656
5769
|
}
|
|
5657
|
-
const
|
|
5658
|
-
if (
|
|
5659
|
-
|
|
5660
|
-
|
|
5661
|
-
|
|
5770
|
+
const appliedByPath = {};
|
|
5771
|
+
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
5772
|
+
for (const txn of appliedTransactions) {
|
|
5773
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
5774
|
+
if (!appliedByPath[path]) {
|
|
5775
|
+
appliedByPath[path] = [];
|
|
5776
|
+
}
|
|
5777
|
+
appliedByPath[path].push(txn);
|
|
5778
|
+
}
|
|
5779
|
+
}
|
|
5780
|
+
const consolidatedValues = {};
|
|
5781
|
+
const lodash = await import('lodash-es');
|
|
5782
|
+
const [currentRecordOk, currentRecordErr, currentRecord] = await tryFn(
|
|
5783
|
+
() => targetResource.get(originalId)
|
|
5784
|
+
);
|
|
5785
|
+
for (const [fieldPath, pathTransactions] of Object.entries(transactionsByPath)) {
|
|
5786
|
+
let pathCurrentValue = 0;
|
|
5787
|
+
if (appliedByPath[fieldPath] && appliedByPath[fieldPath].length > 0) {
|
|
5788
|
+
appliedByPath[fieldPath].sort(
|
|
5789
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5790
|
+
);
|
|
5791
|
+
pathCurrentValue = config.reducer(appliedByPath[fieldPath]);
|
|
5792
|
+
} else {
|
|
5793
|
+
if (currentRecordOk && currentRecord) {
|
|
5794
|
+
const recordValue = lodash.get(currentRecord, fieldPath, 0);
|
|
5795
|
+
if (typeof recordValue === "number") {
|
|
5796
|
+
pathCurrentValue = recordValue;
|
|
5797
|
+
}
|
|
5798
|
+
}
|
|
5799
|
+
}
|
|
5800
|
+
if (pathCurrentValue !== 0) {
|
|
5801
|
+
pathTransactions.unshift(createSyntheticSetTransaction(pathCurrentValue));
|
|
5802
|
+
}
|
|
5803
|
+
const pathConsolidatedValue = config.reducer(pathTransactions);
|
|
5804
|
+
consolidatedValues[fieldPath] = pathConsolidatedValue;
|
|
5805
|
+
if (config.verbose) {
|
|
5806
|
+
console.log(
|
|
5807
|
+
`[EventualConsistency] ${config.resource}.${fieldPath} - ${originalId}: ${pathCurrentValue} \u2192 ${pathConsolidatedValue} (${pathTransactions.length - (pathCurrentValue !== 0 ? 1 : 0)} pending txns)`
|
|
5808
|
+
);
|
|
5809
|
+
}
|
|
5662
5810
|
}
|
|
5663
5811
|
if (config.verbose) {
|
|
5664
5812
|
console.log(
|
|
5665
5813
|
`\u{1F525} [DEBUG] BEFORE targetResource.update() {
|
|
5666
5814
|
originalId: '${originalId}',
|
|
5667
|
-
|
|
5668
|
-
consolidatedValue: ${consolidatedValue},
|
|
5669
|
-
currentValue: ${currentValue}
|
|
5815
|
+
consolidatedValues: ${JSON.stringify(consolidatedValues, null, 2)}
|
|
5670
5816
|
}`
|
|
5671
5817
|
);
|
|
5672
5818
|
}
|
|
5673
|
-
const [
|
|
5674
|
-
() => targetResource.
|
|
5675
|
-
[config.field]: consolidatedValue
|
|
5676
|
-
})
|
|
5819
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
5820
|
+
() => targetResource.get(originalId)
|
|
5677
5821
|
);
|
|
5822
|
+
let updateOk, updateErr, updateResult;
|
|
5823
|
+
if (!recordOk || !record) {
|
|
5824
|
+
if (config.verbose) {
|
|
5825
|
+
console.log(
|
|
5826
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist yet. Will attempt update anyway (expected to fail).`
|
|
5827
|
+
);
|
|
5828
|
+
}
|
|
5829
|
+
const minimalRecord = { id: originalId };
|
|
5830
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
5831
|
+
lodash.set(minimalRecord, fieldPath, value);
|
|
5832
|
+
}
|
|
5833
|
+
const result = await tryFn(
|
|
5834
|
+
() => targetResource.update(originalId, minimalRecord)
|
|
5835
|
+
);
|
|
5836
|
+
updateOk = result[0];
|
|
5837
|
+
updateErr = result[1];
|
|
5838
|
+
updateResult = result[2];
|
|
5839
|
+
} else {
|
|
5840
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
5841
|
+
lodash.set(record, fieldPath, value);
|
|
5842
|
+
}
|
|
5843
|
+
const result = await tryFn(
|
|
5844
|
+
() => targetResource.update(originalId, record)
|
|
5845
|
+
);
|
|
5846
|
+
updateOk = result[0];
|
|
5847
|
+
updateErr = result[1];
|
|
5848
|
+
updateResult = result[2];
|
|
5849
|
+
}
|
|
5850
|
+
const consolidatedValue = consolidatedValues[config.field] || (record ? lodash.get(record, config.field, 0) : 0);
|
|
5678
5851
|
if (config.verbose) {
|
|
5679
5852
|
console.log(
|
|
5680
5853
|
`\u{1F525} [DEBUG] AFTER targetResource.update() {
|
|
5681
5854
|
updateOk: ${updateOk},
|
|
5682
5855
|
updateErr: ${updateErr?.message || "undefined"},
|
|
5683
|
-
|
|
5684
|
-
hasField: ${updateResult?.[config.field]}
|
|
5856
|
+
consolidatedValue (main field): ${consolidatedValue}
|
|
5685
5857
|
}`
|
|
5686
5858
|
);
|
|
5687
5859
|
}
|
|
@@ -5689,24 +5861,27 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5689
5861
|
const [verifyOk, verifyErr, verifiedRecord] = await tryFn(
|
|
5690
5862
|
() => targetResource.get(originalId, { skipCache: true })
|
|
5691
5863
|
);
|
|
5692
|
-
|
|
5693
|
-
|
|
5694
|
-
|
|
5695
|
-
|
|
5696
|
-
|
|
5697
|
-
|
|
5864
|
+
for (const [fieldPath, expectedValue] of Object.entries(consolidatedValues)) {
|
|
5865
|
+
const actualValue = lodash.get(verifiedRecord, fieldPath);
|
|
5866
|
+
const match = actualValue === expectedValue;
|
|
5867
|
+
console.log(
|
|
5868
|
+
`\u{1F525} [DEBUG] VERIFICATION ${fieldPath} {
|
|
5869
|
+
expectedValue: ${expectedValue},
|
|
5870
|
+
actualValue: ${actualValue},
|
|
5871
|
+
${match ? "\u2705 MATCH" : "\u274C MISMATCH"}
|
|
5698
5872
|
}`
|
|
5699
|
-
|
|
5700
|
-
|
|
5701
|
-
|
|
5702
|
-
|
|
5873
|
+
);
|
|
5874
|
+
if (!match) {
|
|
5875
|
+
console.error(
|
|
5876
|
+
`\u274C [CRITICAL BUG] Update reported success but value not persisted!
|
|
5703
5877
|
Resource: ${config.resource}
|
|
5704
|
-
|
|
5878
|
+
FieldPath: ${fieldPath}
|
|
5705
5879
|
Record ID: ${originalId}
|
|
5706
|
-
Expected: ${
|
|
5707
|
-
Actually got: ${
|
|
5880
|
+
Expected: ${expectedValue}
|
|
5881
|
+
Actually got: ${actualValue}
|
|
5708
5882
|
This indicates a bug in s3db.js resource.update()`
|
|
5709
|
-
|
|
5883
|
+
);
|
|
5884
|
+
}
|
|
5710
5885
|
}
|
|
5711
5886
|
}
|
|
5712
5887
|
if (!updateOk) {
|
|
@@ -5891,6 +6066,38 @@ async function recalculateRecord(originalId, transactionResource, targetResource
|
|
|
5891
6066
|
`[EventualConsistency] ${config.resource}.${config.field} - Found ${allTransactions.length} total transactions for ${originalId}, marking all as pending...`
|
|
5892
6067
|
);
|
|
5893
6068
|
}
|
|
6069
|
+
const hasAnchor = allTransactions.some((txn) => txn.source === "anchor");
|
|
6070
|
+
if (!hasAnchor) {
|
|
6071
|
+
const now = /* @__PURE__ */ new Date();
|
|
6072
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6073
|
+
const oldestTransaction = allTransactions.sort(
|
|
6074
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
6075
|
+
)[0];
|
|
6076
|
+
const anchorTimestamp = oldestTransaction ? new Date(new Date(oldestTransaction.timestamp).getTime() - 1).toISOString() : now.toISOString();
|
|
6077
|
+
const anchorCohortInfo = getCohortInfo(new Date(anchorTimestamp), config.cohort.timezone, config.verbose);
|
|
6078
|
+
const anchorTransaction = {
|
|
6079
|
+
id: idGenerator(),
|
|
6080
|
+
originalId,
|
|
6081
|
+
field: config.field,
|
|
6082
|
+
fieldPath: config.field,
|
|
6083
|
+
value: 0,
|
|
6084
|
+
// Always 0 for recalculate - we start from scratch
|
|
6085
|
+
operation: "set",
|
|
6086
|
+
timestamp: anchorTimestamp,
|
|
6087
|
+
cohortDate: anchorCohortInfo.date,
|
|
6088
|
+
cohortHour: anchorCohortInfo.hour,
|
|
6089
|
+
cohortMonth: anchorCohortInfo.month,
|
|
6090
|
+
source: "anchor",
|
|
6091
|
+
applied: true
|
|
6092
|
+
// Anchor is always applied
|
|
6093
|
+
};
|
|
6094
|
+
await transactionResource.insert(anchorTransaction);
|
|
6095
|
+
if (config.verbose) {
|
|
6096
|
+
console.log(
|
|
6097
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Created anchor transaction for ${originalId} with value 0`
|
|
6098
|
+
);
|
|
6099
|
+
}
|
|
6100
|
+
}
|
|
5894
6101
|
const transactionsToReset = allTransactions.filter((txn) => txn.source !== "anchor");
|
|
5895
6102
|
const { results, errors } = await promisePool.PromisePool.for(transactionsToReset).withConcurrency(10).process(async (txn) => {
|
|
5896
6103
|
const [ok, err] = await tryFn(
|
|
@@ -6654,13 +6861,15 @@ async function getLastNMonths(resourceName, field, months = 12, options, fieldHa
|
|
|
6654
6861
|
|
|
6655
6862
|
function addHelperMethods(resource, plugin, config) {
|
|
6656
6863
|
resource.set = async (id, field, value) => {
|
|
6657
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
6864
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
6658
6865
|
const now = /* @__PURE__ */ new Date();
|
|
6659
6866
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6660
6867
|
const transaction = {
|
|
6661
6868
|
id: idGenerator(),
|
|
6662
6869
|
originalId: id,
|
|
6663
6870
|
field: handler.field,
|
|
6871
|
+
fieldPath,
|
|
6872
|
+
// Store full path for nested access
|
|
6664
6873
|
value,
|
|
6665
6874
|
operation: "set",
|
|
6666
6875
|
timestamp: now.toISOString(),
|
|
@@ -6672,18 +6881,20 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6672
6881
|
};
|
|
6673
6882
|
await handler.transactionResource.insert(transaction);
|
|
6674
6883
|
if (config.mode === "sync") {
|
|
6675
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
6884
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6676
6885
|
}
|
|
6677
6886
|
return value;
|
|
6678
6887
|
};
|
|
6679
6888
|
resource.add = async (id, field, amount) => {
|
|
6680
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6889
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6681
6890
|
const now = /* @__PURE__ */ new Date();
|
|
6682
6891
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6683
6892
|
const transaction = {
|
|
6684
6893
|
id: idGenerator(),
|
|
6685
6894
|
originalId: id,
|
|
6686
6895
|
field: handler.field,
|
|
6896
|
+
fieldPath,
|
|
6897
|
+
// Store full path for nested access
|
|
6687
6898
|
value: amount,
|
|
6688
6899
|
operation: "add",
|
|
6689
6900
|
timestamp: now.toISOString(),
|
|
@@ -6695,20 +6906,24 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6695
6906
|
};
|
|
6696
6907
|
await handler.transactionResource.insert(transaction);
|
|
6697
6908
|
if (config.mode === "sync") {
|
|
6698
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
6909
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6699
6910
|
}
|
|
6700
6911
|
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
6701
|
-
|
|
6912
|
+
if (!ok || !record) return amount;
|
|
6913
|
+
const lodash = await import('lodash-es');
|
|
6914
|
+
const currentValue = lodash.get(record, fieldPath, 0);
|
|
6702
6915
|
return currentValue + amount;
|
|
6703
6916
|
};
|
|
6704
6917
|
resource.sub = async (id, field, amount) => {
|
|
6705
|
-
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6918
|
+
const { field: rootField, fieldPath, plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
6706
6919
|
const now = /* @__PURE__ */ new Date();
|
|
6707
6920
|
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
6708
6921
|
const transaction = {
|
|
6709
6922
|
id: idGenerator(),
|
|
6710
6923
|
originalId: id,
|
|
6711
6924
|
field: handler.field,
|
|
6925
|
+
fieldPath,
|
|
6926
|
+
// Store full path for nested access
|
|
6712
6927
|
value: amount,
|
|
6713
6928
|
operation: "sub",
|
|
6714
6929
|
timestamp: now.toISOString(),
|
|
@@ -6720,10 +6935,12 @@ function addHelperMethods(resource, plugin, config) {
|
|
|
6720
6935
|
};
|
|
6721
6936
|
await handler.transactionResource.insert(transaction);
|
|
6722
6937
|
if (config.mode === "sync") {
|
|
6723
|
-
return await plugin._syncModeConsolidate(handler, id,
|
|
6938
|
+
return await plugin._syncModeConsolidate(handler, id, fieldPath);
|
|
6724
6939
|
}
|
|
6725
6940
|
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
6726
|
-
|
|
6941
|
+
if (!ok || !record) return -amount;
|
|
6942
|
+
const lodash = await import('lodash-es');
|
|
6943
|
+
const currentValue = lodash.get(record, fieldPath, 0);
|
|
6727
6944
|
return currentValue - amount;
|
|
6728
6945
|
};
|
|
6729
6946
|
resource.consolidate = async (id, field) => {
|
|
@@ -6809,6 +7026,8 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6809
7026
|
id: "string|required",
|
|
6810
7027
|
originalId: "string|required",
|
|
6811
7028
|
field: "string|required",
|
|
7029
|
+
fieldPath: "string|optional",
|
|
7030
|
+
// Support for nested field paths (e.g., 'utmResults.medium')
|
|
6812
7031
|
value: "number|required",
|
|
6813
7032
|
operation: "string|required",
|
|
6814
7033
|
timestamp: "string|required",
|
|
@@ -13391,7 +13610,7 @@ class Database extends EventEmitter {
|
|
|
13391
13610
|
this.id = idGenerator(7);
|
|
13392
13611
|
this.version = "1";
|
|
13393
13612
|
this.s3dbVersion = (() => {
|
|
13394
|
-
const [ok, err, version] = tryFn(() => true ? "11.0
|
|
13613
|
+
const [ok, err, version] = tryFn(() => true ? "11.1.0" : "latest");
|
|
13395
13614
|
return ok ? version : "latest";
|
|
13396
13615
|
})();
|
|
13397
13616
|
this.resources = {};
|